{"meta":{"start_date":"2026-10-07","end_date":"2026-10-13","start_at":"2026-10-07T00:00:00Z","end_at":"2026-10-14T00:00:00Z","timezone":"UTC","generated_at":"2026-10-13T02:55:06Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":311,"failed_runs":238,"raw_occurrences":893,"matched_occurrences":875,"runs_affected":220},"rows":[{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":149,"runs_affected":37,"failed_runs":37,"run_impact_percent":11.897106109324758,"anchor_occurrences":149,"after_last_push_count":195,"also_in":["PROD","STG"],"trend_counts":[236,126,12,9,2,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":207,"prior_last_seen_at":"2026-10-10T02:02:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with a private key vault using an available API version","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":15},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":10},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23a798e4e3a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7kr9hzf26ffd\" to finish creating: GET https://rp.j0742656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"name\": \"2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T03:18:06.524337179Z\",\n      \"endTime\": \"2026-10-10T03:47:07.577893958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23a798e4e3a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7kr9hzf26ffd\" to finish creating: GET https://rp.j0742656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"name\": \"2a131d64-b48f-453b-b60d-f4063709a357\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T03:18:06.524337179Z\",\n      \"endTime\": \"2026-10-10T03:47:07.577893958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-nk7dk to exist\\nMachine p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7kr9hzf26ffd--managed/providers/Microsoft.Compute/virtualMachines/p9a2k4m8s9w5v9a-np-1-b4xfk-vck9z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x560689fa080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jzlxwvqzg694\" to finish creating: GET https://rp.j5842176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"name\": \"efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T02:25:01.899620825Z\",\n      \"endTime\": \"2026-10-10T02:54:06.245842706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x560689fa080\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jzlxwvqzg694\" to finish creating: GET https://rp.j5842176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"name\": \"efe266bd-17ba-4b43-bc21-b2099c2f07cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T02:25:01.899620825Z\",\n      \"endTime\": \"2026-10-10T02:54:06.245842706Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6u6d5o1h9k6y0v-np-1-wpskn-72d9j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jzlxwvqzg694--managed/providers/Microsoft.Compute/virtualMachines/j6u6d5o1h9k6y0v-np-1-wpskn-72d9j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3113509f94c0\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-g776ncnttgrt\" to finish creating: GET https://rp.j3788032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n      \"name\": \"77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T19:34:01.982763751Z\",\n      \"endTime\": \"2026-10-09T20:03:10.160667249Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3113509f94c0\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-g776ncnttgrt\" to finish creating: GET https://rp.j3788032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n      \"name\": \"77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T19:34:01.982763751Z\",\n      \"endTime\": \"2026-10-09T20:03:10.160667249Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108739904140742656","occurred_at":"2026-10-10T02:02:26Z","signature_id":"899ee92e21aa873413f2b225dc269879ee50d989eb61e324bf237f0a792412c9","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108729709905842176","occurred_at":"2026-10-10T01:21:53Z","signature_id":"0b3ceb5461aa5f357bf0cb7f974c95380ac2a402e404e4401b031539454edcc4","pr_number":7177,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108618645423788032","occurred_at":"2026-10-09T18:00:35Z","signature_id":"ecb11b0d98366bb57461afb65e5a662d355410505007e7cb21a482baec673750","pr_number":7427,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108538835301306368","occurred_at":"2026-10-09T12:43:27Z","signature_id":"3391ca8dbe2cfc6a644473fafbfda82e44265275df5a1354a12ae66728b85887","pr_number":7401,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","signature_id":"b153d83c712b7a426cea9f27f54a827e240c35a8f024ef0bde94a81ade36c37b","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","signature_id":"b4cf9fb75f372d37e5b213fd1d972d3a7a4a75135dc7f43f2e40572b1ca8a71b","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","occurred_at":"2026-10-09T08:49:50Z","signature_id":"4a88a8d89596c34e4f3ac28e02db74d5d14c30f8361d3164afdc9f87cc8dcc79","pr_number":6944,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","occurred_at":"2026-10-09T08:49:50Z","signature_id":"ae60c1c69c20f023c68844b91b0f521ca18d833de14f1dfc77a24752e434f5aa","pr_number":6944,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","occurred_at":"2026-10-09T07:02:15Z","signature_id":"e6756c0b7d7401645c66fdb6cab5ac9e0b763efab297c59c3a9dfd587ce7507a","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7438/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108402844888469504","occurred_at":"2026-10-09T03:43:05Z","signature_id":"b15702fdd5528fda136f82def4e6c124ec913c615faaa8a57b58834afb1e560b","pr_number":7438,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377905737240576","occurred_at":"2026-10-09T02:03:59Z","signature_id":"6d149adf9696efaf8354e5d9b60fb40f03481cb87d334b82a607b6fd552d916b","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","occurred_at":"2026-10-08T18:12:42Z","signature_id":"9bbb9a304b3e216bd163fb308a5bc70d9beeba6ca91eeefd3ac9063123943dc7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","occurred_at":"2026-10-08T18:00:26Z","signature_id":"e52340d4b382cdd2542ada160fb67461b260ba9f52c641a312b0e9e87e132a0b","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108220296300335104","occurred_at":"2026-10-08T15:37:42Z","signature_id":"d5485ecbaf29eb6153d7283048b59ef21ec89644b21cb87a56aa54d40e6a3fa5","pr_number":7431,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","signature_id":"9f5e90ecbae9c619ce85e90de1ac44b213ae43628780d197bc24085fef9e391f","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"6089f291ebd2776303b1bbe50c4d8aa3d633aed985bf844efdbe7aacda4c2397","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","signature_id":"1cc5d051835a1c8eb905dcafe82dc37af6a69b3130687b92e4a9ee1058c03a32","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108129603208876032","occurred_at":"2026-10-08T09:37:19Z","signature_id":"b1cc99febe4b197af72ea8a773b460d06bc2296c0c3a577338c7f8d560c7e86f","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","occurred_at":"2026-10-08T09:09:57Z","signature_id":"167a399ee5497032fbc211b91c5977c4d716b98bb1d5acb6868aa44e16e76145","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","occurred_at":"2026-10-08T09:09:57Z","signature_id":"8a3ed87cf84ff42b79a1fd8e8f951dc027047229d3c301b92c6142a018727cb3","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","occurred_at":"2026-10-08T08:16:51Z","signature_id":"f22d3abbd1cab4d50eb4fc487acaabfea9c0b84dab46bd31dda1987b2400c11a","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","signature_id":"5fc986fabe0026f4b77c8908e8c0e313a1116526b8618c4e0b755089134b6b09","pr_number":7412,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","occurred_at":"2026-10-08T03:54:28Z","signature_id":"f21ba692b551caebe9fb47bce79bd94b5286f66a20a64edd5fe17a6b67fb55c5","pr_number":7177,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107939651057618944","occurred_at":"2026-10-07T21:02:31Z","signature_id":"e81efffea3ea1b938a7a9d06760cc35a55e43a70882135723090d05ee5ffbc65","pr_number":6954,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"4afebe6cfa4becd64f3753581a33be535c50f0fc8ec471f7b9a6903b11fe2376","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"18f2e28a07e5eb92679cc517800744e5df3a152e2396fa940c051e7574885e9c","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"1f0e4f0c636adf2b731dd9281d3e7bb9c1c0e2048ac2b45a87fbf99ecbe04ac8","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"2926fb9d7060c87470cd426a61c704bac975307a608d031bbdacca06b4fc644f","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"4bd14a829b89ede2b97f05bf6929b7869177f362d89becc37f9d90e292a667b5","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"4dfde317b11106ca0bd23bf0cc1be0c00200032488cdff73f9587e90545288f7","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"76b920439a80aa38a9cd0875591b6edf3e0e9af781b1c8b022ae07a8fd86dbde","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"d076bde588818dceafe9295def496261167c5e627cf0dfa6e7e59d7fab7a0d26","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"df7306baffa1f0284a685605b0198dcfb8e6e6fa960112f72942b626ce31a529","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"e1f2efe03af68d4ce31e31e7c6ae84a90a95274374b7894b8be874df43aeb960","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"1c481c423bed0c16afe7ba8ce852af80db737e5d86b88d6594b69a5b2349734d","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"1eaca06a614138032740bad113fd6bc95fad2738fc2fec29290c9c84eb0162ce","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"4e7bbd17a69e19a522e72b5689c49118c5aad05be581a2713696272f925c70fa","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"52eaceaa1b46bb9ea1f0c0251416cb3da7851bb61c9a6e5565b5649beaf12298","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"69e74bb1a59bef22e2d3c9d8db69756b720248ab7d712391101dd02a7c60b033","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"75fa85c6eead75af9929e7654b5491890b965767e198093cbe942d232b71c61a","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"79dcae13bd7e35f5ef9c3691d9e87c155c8a8b1248ad3ddfff42d29fc452895b","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"93fb0fc62da8fba90112959c0e5524d7d28a492d00287a557d4996bd704526e5","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"98b46fb6c08d77f3f18d038b41ae82a882fc3d4628819ab2e9ea75b061cbd288","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"ac773143bc795433482c697f6cbd0e48f70363b2467a2da7e9de3148f8e2cd51","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"c967cd94b64d80ae0cc4a8a5f628e078837c44d0d953179ca8808c680873430c","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"cd3df0ae09c9ca011557e453c48b836e776c10198a4ff32ec4aa8ae20b7ae1a7","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"d7d9bc5f5ed38e25615efd7d9114e5c09fd421668e2a82dbf96d7fd13eaf6419","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"f4cde9518c1a88ddb8f822b81b4d7416f72ee540e165dfd3aae64bb95d75b1ec","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"110022798a0764c5024f8b28297ceca546ce14d0a1c6b612a61892cb5628f7a6","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"1b93fe9594667d36e6a2a6bd0616e1d6b11af83f9031d6f6ca44578ccd57caef","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"298862f500cc93b77a8c824b97595e4e37374b181407fc37157fd10b1e4c8c31","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"4c030cd147dc6473f9050e85fa4711eae9edb3e863c902974c185ecef7f0f633","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"4dfc950127ce86a016e64c08dd03052325b83c724046c516f0a0a51d07e6cde0","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"9199a30bc09dac1a956a7174474cc1782b878ab9069af756a8200e5c362e4a72","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"941be88d29bc7cff4b20af1db30253a5b8849dd7ba4250859ec3fffc8398f027","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"e5d7214ecd053506cec60d6039d445ec28a5ce3412453da44db155132462f979","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"fbf7fec41b640eac5fd46e05db1c159186d578dde36fe0f54035b78e8ce546f1","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"0fb6e1ec1717e82374bd0ab14045070f740a319efcfe84a5e1624839087834fb","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"59295f394d47135b361b2ed8985d6bc279f8687f208e575aab5859cb7bf31166","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"5d3064cdf6d821edd7ecbc32809672863b0adc7cf3ea6df16687482e8b85a923","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"74bc64dc70ad0017ff4f9b03b43959fd5f11246d6a15f16b814179375af6d735","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"99eb96d1396ba54566b3369529aeac2c545ab5eb91f2f84221e8279e27dafbf7","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"a76baf01b55b7a5ecd74c09bee77d30a3af38608bae3a4d5bbf78b1d074f8b8c","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"a90c96add63ec094880f02e255876531dc36c79c1f8d294ebdf378e9792415fc","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"c396caa9bcf0c2da1cd4662a8eb438222950ad162bc060be8a1addc8671b604b","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"c44c4d07f31f7026d27303c689b6f97d8b8cf333b83abef19daa07d09700c1c7","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"c97661e923fa1de8c63c33e700c6ecaf4ddc5c0588823453e627421ad9c0079a","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"0cde671c6e56522b67cc12ff4f25c7e4f0555299340490ca66e92acefbfbe1c1","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"15eae028ce53f84d13e63bb41284ec43f8011d73906a47c6d01bb1d26619a589","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"1602dd762444352f2f553c13ab6b7ee18d842ad9ec9a8b7c6c869612269efdf7","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"1bbd7ef56debff6d5c7488c79402871459f0f003e9957dbd25aa475bd5d012d6","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"2350e506ef8a86a7ebd95909719dcdd3df80ab16e1752475b084c284434ab98d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"29781840512547ebdffe5b318cbf69d0eb9c139f7dd9a163309a42fd1518be5b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"3530c780c67a7e45eb884abbc05d52623a1cac06fd06d9407918744d15ce231c","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"6a98eed0f9f19d9b2606b420b4aa3c5f2ee0276ffd1335f89b2c5994956cf6d5","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"e4197d8f95eeed95cbec563c3d542bc9280009076d63cb82bbef9262337cecb3","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"ea20ed5ededb7561ed93b9f6df6f5b9b4077d95aa023417f6ee9312af308cb2b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"05d8468ee56d53938c0f340e048e58d034a108f3f7a32001325ecf37dfe50d35","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"07a2a756e0c8a1bfef77b8bddb555d9e1545f1c96712630cd70c2290f204e24b","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"0c3e6b55ebab2f04a60faf61e054837580d8faa3d83e2cd44ac9868c9cfa8838","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"7eddaa92f651d8945d08fb4abbcb977dee0f718b2c4423dad1308f673ac0ffde","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"e4fa04f3828672393bb8ee717cf4607e3a078303082ff4844aa7b2729a55ffc7","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"ed20db877f6a2fe61c73e70aec12091b55b232eafae5e6f3a702af1d0ace5203","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"f5be6433f057fa85560c4273cfab71f2f16027337caee499a075f51ad97a2b5f","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"3a098b48b06940d26e0465ac20220cb5b7e73dd045646f72f9fbeeaccfdbf533","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"409c159a14de870f32d97bc50495e35950f682f737ef9607619128669adac534","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"5cf2a5571f1f56c265ec37f0381a1c8ed4008bb85ec06c4ae7cf57d61c36751f","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"60baf98859df1e164b9731da33e2dab18a29baa7c0dc9152363e95d9b3d8e605","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"843f6080d43311a72cff38f636f61f742f4dbbf3764932025820682bab88fbe4","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"9777b9adc2ed60d49bf0bb6bee7a613f4b848f9ce3762e434e392fb166d6f760","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"bce41bb751b8a42990d8eca8acf4cb9ef8037f2022db5e9570213dc8295a97a2","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"bdba72a5bf9a4e6a7d12a795d000a3b484b9987a912e0448d53f34ae618cfccc","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"2132eda40301f5eaf21d1b30205a9a5574c75a30280f0494851dcc8cf854d03e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"5359b5c88f9e8c98720978868a5088458008510399612cc6d979cfac3938551e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"750bbd1284409385baf6a12c22a9104af08af1e526abad4017570508d77bc277","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"936557c8acaeb1d7461dad4a5705bb52cb39c2b58ebc33b681671d3a2176e5ac","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"9aab43aa1fad33c45c2d9c115547e491f171c8e5dd8ebba2e023ceb28f5c022b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"b2e36904ecde082df7969eb49ba722cc1ebf740798096de32b1dabdda712327b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"c05743641b2b3d4c262aaccee2431126664657c73fe13d93acdb1b842b38ff45","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"dfc3f987c8a57f6e0531aa5fb1800c718ed85dc061fb6602bb86fe271cc56f17","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"ebffe6d6ecaf5e5873fbc8ae713b30c13868e7b43d7e1847d51ebfcdd987068b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107778619966230528","occurred_at":"2026-10-07T10:22:38Z","signature_id":"45abc7a27580a0d545fb5071d27064b3f18f5dbe32902c6602415496760c90ea","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"18b3fa7498379ee80ea560959f2e6233124f9b4a98d1ba7ce94dc24c4bd18bfe","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"28fd67d083a935f18b35824e91c321d3dc0a675a33cc2ad3042f20f7f21b2d18","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"29826f2dc72b792b387cb9569abeebe12cee5fb64b9656f64489111f62e4d0ec","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"39844fcc919184e3e26c50c19f16c1f74b28fd4bf3496ca74f197fb5b19b6431","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"39c083e3464a733a1377083d50da650490486daa38d016e0263bd9f0dd798841","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"42775d6d590be883e5cfd0afe00661abe77091dd685a801ceedb8574ff190602","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"84298de935fa96ecb1c7c68d067ce795f5ad49d83489fd349fcd2e4d24b108ae","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"8d589358c3bb0331b165597c6620eb42b56f46ffb021a70676a3457b789cfadd","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"a0edcc1432c7ec386284464c6dc9b384a6b36d43976ceb410c0dc0b24fec7f48","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"b06bc26557c2f7b7c1431f21a94f48d01c18df7244e082d80bf24ef9e3a7679b","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"c5054e34b231cffbef01421a9b31fafd7bd6e6472dd05a5f0e6b5ec197ea182e","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"05b82e80d394db855f72820095695dccb352d1ff5fc4eed9b28245e606c0d145","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"10c21f1ac4badc1c51061196b70c294168dac2e4ae5c1b9ec1c48bbe0e84ed74","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"3462fcf35a7e1ba371569d586044ac77401b9bd3ec7e4f2e4b22f1363c76be6d","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"50af757e09db672b178bd70b1adc1f876736fc58faca8db721bc68bbb1c417a5","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"696ec44a6df12b95105a2bc29ec5070faa7f6f41ba0f527c2fe0cd86b769e0fb","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"6f22a5b326045bc8517c0d65b469434082b3598a3c311cea397143588ae18d8e","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"c1cf9eb7afd6e93fb115c36a108273eeec62e2d5e0189af0f151c3e257eae80f","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"db54fa3dbd715eed9d612926ede9e164fd802289941471af853ad4a488bd084a","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"1add0a99393798752a0915fb4ea70862a58bcd3e033df28f52529f24f2f3a585","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"2170f480f911282c2abe5dc130e3b47c1b6faceca498b12db69544508b616cf0","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"2a51981b027d8aa97e4732f876ec757c350d36ec76746d72814156f84c2dfbdb","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"561ba19fb10ab687391aae0274302bec54070c7d29409470414b7dd1608f51f2","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"983fd3933dce71eb0669b8069c2aca9caa1b5d4aabd629fd7933c8491dc60b0d","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"a902f649acef68c24fbf4ded739d69eff3baf267586aaac8e393a75b25f0662f","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"df226f17583426298a4db5629842c56e1505c2567993c63de76bb3dd6c98edb0","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"e2a1db1bc07a51865a6d72643e3b66dfc46c1c837e30ee4392486f8f885f8d75","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"831b58931b42de25f29f96356e7160819ea0460206c7caf7b7aecd3a79744749","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"8cd5dff5cb797cdb21d04834eb3a07a380cb9ba6d257444312311bec337f797c","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"c8c45cb1becca3e1c26ed358851cc615189db8761433406668ac8b4a33f3428c","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"d81d269f8740f219f8eda6fcdc0709341c1d9b64d4c791777028838ce00216a8","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"df0ef413bf45345efe9139d1eb01f7179a89d40a680881f1fdcf15ebb2bbc0d6","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"f53952dfa7ee6849ad3e00f8bd97110d104593bd40f0b476fa6c5dc473697253","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"fd080a7568bfc33f70af0817e2c46e2e7ffa2e4df4898a7e33343134e0ec60da","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"fd5772fdac77f3ca1924151a6f31be6a24c58d4df40c7b6bbdb288bea1c86f68","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"319ff80a119cce276594c97e3e93d4509ad81c0e28a12bbc3e8fa2a597d14dc9","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"115fc9a87c835081d6747cb81d8e5f2ce1c931210978e133e13a437ef181c0f0","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"14d6394d0f9b51519a75c1fae3e72f083c452019ec69868fba057c34084a74e3","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"4a10651c6d339aec715190ab280a112cb4e3daa6943c976a5ab843f01fa2bf0e","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"6f1def9963421cbf1384fc961c05ecb86c555deca9a956b6cd51244bac971356","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"90299ee13eb97d04cd02aad44ba99f8edfe16f2814bc827bcc704513ec15ae9c","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"953b2c19892b92d0fc0c3df69d146d6f47f763be247d3757622c054f4b956a2d","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"95a1e8417d0a16f010d78b2112fce589c5e950c5099fee6d99af7f39eb186080","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"ac8e4fbbcabf95cc130fe5b4fc037bd85d7b0106da3b7a49d23bbaba2d73569a","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"b8a0763536367e04cd9eb8cf0ed07c7979f2cde3281236d8e855a4932da00c7c","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"cfb3c0aeffb83e972da151272b81e096b174adbbdd1f6621c8ee47c5a32d001f","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"ee750420154f79a00dca045668ab90bb645a634d33016eee21f6ea06864c7b62","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":135,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":135,"after_last_push_count":1,"trend_counts":[0,135,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2f8378088558\u003e: \n    resource group \"kusto-logs-zdcz6ngd7l8s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zdcz6ngd7l8s\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"name\": \"ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T13:55:49.637158017Z\",\n      \"endTime\": \"2026-10-07T14:19:49.807578144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vu3tto21eb5vs2oeuv4d035r4kh9q still exists (deletion dispatched at 2026-10-07T13:56:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2f8378088558\u003e: \n    resource group \"kusto-logs-zdcz6ngd7l8s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zdcz6ngd7l8s\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"name\": \"ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T13:55:49.637158017Z\",\n      \"endTime\": \"2026-10-07T14:19:49.807578144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vu3tto21eb5vs2oeuv4d035r4kh9q still exists (deletion dispatched at 2026-10-07T13:56:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xf860122e060\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d7rnngth6rmt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s5wrnm\" in resourcegroup=\"rg-zstream-upgrade-4-21-d7rnngth6rmt\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"name\": \"de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:14:15.375309734Z\",\n      \"endTime\": \"2026-10-07T14:38:20.14126941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtq0dj9uuqrc354smigdjo5gckvsa still exists (deletion dispatched at 2026-10-07T14:14:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xf860122e060\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d7rnngth6rmt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s5wrnm\" in resourcegroup=\"rg-zstream-upgrade-4-21-d7rnngth6rmt\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"name\": \"de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:14:15.375309734Z\",\n      \"endTime\": \"2026-10-07T14:38:20.14126941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtq0dj9uuqrc354smigdjo5gckvsa still exists (deletion dispatched at 2026-10-07T14:14:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3d715b6dc0c0\u003e: \n    resource group \"admin-api-serialconsole-kgtb4656t7wx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-kgtb4656t7wx\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"name\": \"63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:05:26.627856798Z\",\n      \"endTime\": \"2026-10-07T14:29:30.238184303Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtu5p75nk86f54m2uds6vreosd2p2 still exists (deletion dispatched at 2026-10-07T14:05:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3d715b6dc0c0\u003e: \n    resource group \"admin-api-serialconsole-kgtb4656t7wx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-kgtb4656t7wx\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"name\": \"63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:05:26.627856798Z\",\n      \"endTime\": \"2026-10-07T14:29:30.238184303Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtu5p75nk86f54m2uds6vreosd2p2 still exists (deletion dispatched at 2026-10-07T14:05:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"00770c4fe97dfc58f45740186e91ff796f07117f2e217de3f77804d73f958c0a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"013b36a17be5e1755039e5e47a5699ae4977c8c5bad7e790f67e10084353346e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"029c4854e61d0d203eced163c848473e3858d9fb8f5715fbb1ab826a86732057","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"02b669fab2f3a752654dc9a34f097918516a7c5968bad6ff55e521951f3af1a4","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"070933d7361dc14a21e4c08bdac9bd6cd2038d91232c2cc7958eb579eb093158","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"0e6e93a01ccbe18930c88f6800d0218371948c8b0fcbf57358cce8f18990f3a1","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"0e7b9a1304f6b5950ff5a6780c8569a77bfb9dd61b6ace6197c47db81e546ab5","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"0f75db649ea9fa6db01a02d3b5637d779ce9e29da1a572be08081e8d2fb66e8a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"14983add3cbb52204586976e52460dc077bc8f9f1bba2975dd821355d1384070","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"1bc6da96879e0d6f1e2c7c45a23431c8c4f6454dec82e919b9545e6949a18f21","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"1be34856b46f8d6a9c44093844c8700f3269be9e60a78622e9533e8933047309","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"21c48ae3defdd43c7577c013e6f6c9fdd335dbfe0b89683654111c1b5ed39e86","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"26681a4be74afffb2e578b6b4027025365128e4c366f444bc762944b6c1b93df","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"276eea38d768faf75d57e95a2137268e1a8a0f332f48a71f2d800f4634e514fd","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"285faa2939852d449718aa4c429df428b96828f95653c2e3cb13ef94e108c9b6","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"286cafcce7f2030ffc52038d15d989fbe508aa9222743b5b8ab4a6c9345e520b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"294252fbbb161d3da22685c5b51e11a3f5e1fab801d5d967f03d3d6ddddbd6a0","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"2cd950718431e65c60c1449982bcf45b52fc9aa594f2e7f4733dab853ecd4237","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"3435cef6023d2bba4e541a2c09290178c6c9c1e592a41ec1fc2cbe0afc333879","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"3ed6e29ae24b22e936c139bddd29259000074bf38703eb0fdb9831e5f87da761","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"45fa548a5e7b5c65345ec02c8e2ab623bbe96ea387f003a14bf0fe82619d5276","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"48e55fcbb84d11612bdbc4e6665d9c301500141b07dfb62e29d7676b97884aca","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4af3a70cd20df1fb98271c331f06b303636ded7bdbad6e22483d028f110d6c72","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4b9bc53d8926afbba2ea2d19edb4820af2e72e4808254b23e4b76a910553f9c6","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4d27311f1e6a388347006b63a44e786c6632627915c8eb4e51a9fd9526df314b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5575d92252fa76c383dbc5dd7c06851a1356391e4e7aa3ca4aa803c9d3683f26","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"560f0c3953616fa100db0341bb4bdbea0d8e6993955774de086a0dcc2ed1e90e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5743dbe7878ad72b0dfce27b3e213f15fac28467e47138f4c8afcf36a31c1ffb","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5855b5533b4e07a4cb53c7524cbdc5aa4a741605c71ee80ff83ba4cbd790a596","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5ff85e07f1038463bdaba2636a9491acb46a277550d9d392e9f98aab2a43d79e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"6442fe24306b709cd2915825463d36c18d25267ee1bd3c6a3fc8d3f372602919","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"6da31d750be1af3dbbe6e38b503e19dc84ce7f67730e4acb47ca85c9a3997672","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"6f5f7a1355e4ed73e51bb3f030f677880809624e0f8b786c393a351a2501219c","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"75c5bc63c99001fb0f2e8b2198e551808d3f6f57309385f288b8d942634a1c35","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"7df2c0e95ea58a049ed830fcca9c9880a7128409f67c869b53eea9387b2285da","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"82e73773d549ba957f2204af6d4d2c1cf170a7a161c00e04165cb68496e70837","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"86c60664512b45dfc964ed34fddadfd8c59b67915c486ec96cdd7da4f6c95f9c","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"8f9144d292126dc35d2c1f9809907292f12b4514db461a32b70a7d837967042e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9a7da0741b30aa727c3e373095e1a0dd725bae4e61b8eb5638e69048b7203253","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9cd0816b849969b8416e43fcacbd8a3afe33dd8b72e802c42c355de1252b8ea6","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9fcb18d4baf46207c53d60b0635afad4482714533890bb48e436a254a5387cb7","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9fd8fda4123d9907523177f00d190d89e7f1ea8c687b4ca80c64e35cfefd2f5b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"a062845896766f82b5b6b43caec725177925d11140bb3f164d4f624bef40db4d","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"a5570161b8e3666ca7bfdfe88f809978e21dad7fe68a98b14aff3c246574fa43","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"aaed1413dee4877e1b256246048dd53e1f6b2dfba7ace865bb05935e5ca237b4","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ab86b2bd678ab3ef10ab68c466b7ecffe23bfb3a6ef5dda3abadfbfe4eaf9f55","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"b5b4a6f0378c43d1a2dc12c93438af0fd66b0a976463f2bebecb04c0d2a2965b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"bc974c6166c1205885fd7f678176787dbd1c4bec6c3b41b99aac04cf4e8b0438","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"cd677032a06805aa6da80e46d98a979f1d4459f91cd151d2bd701d7f53300c7d","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"cdf9bccd98bb3d1f4fbecd496356b9b4b2a6909b9cc5bbfcc35a4c6dec0be79f","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"d33db4510aad5ad39a18b96587e435da48fd02ce7f5d805a2a95ddbed99c55be","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"d844d68bf4a7e9dca03bc30788bc19d0b4bed38962bfc0a6b87e74a5926bfb9c","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"d971f81ea2c74ac5682a34fb0a1da00c06cc6e232238cf80a72d313131438d96","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"dfdb82b5e1768bb75ad1b5945dd0d4846e83397c37919c97b3cf70301d4beee0","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e05c6379c4a0811c95a6f224b6eb160bc7e8155848198ef8d53fc4e1e641d681","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e55ec4dc2559e274b43ba58584e2196b6488167e7b7f10b0081e0a9c39f29e17","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e6cca4e5ebe2824c588f245b4fcfc45c438b672a120705e75673912bad4df3ea","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e97f5f31dc8603bf599703c5d2bb6392233b12e869441a56869a6ed9f3cfe6a7","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ea136f16cccb77cddc98cddc0a14a83b77bfecd278fefad03002866b67a2a785","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ecd414defcb93a1a62a5218474be07f4e6bdc602b84512c4e5b6a4c046f93ab1","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ef7e28288ab5d242a97c713ca8d4978be14e4746798d8fe8f1199a256df987ad","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"f7448a3ce1b674b582afc3ce637cc18aeda6a4b87548b38419f0fdbc6a961f0b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"f8d57dbc64ef0cd306ea8787bddfb2291508c002358648a4f849e60b73624f16","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"f93f45f1ba356291f8e43d864d0c95ff1aae399d7e3555e18bb2fb08f56aef85","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"fa6930c5b5762130b890761a4a2f2b159ebdf252501d3b747e67c32a195f1212","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"fa98bbc082e6f0398dc6e7ed99f2366586b19c3e07e14afe3e1eac22a02a9e8a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"fb1584ba45514cfa19ff8297d90b87cce89225faeafea847902ff47540b55603","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ff35b990ee10a0b7d2f378b4f77e12fee7a2c01cba6e29d35fccf29a3fc32400","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0234af5f580db6626f2a7da3e559fea14c85fbeae8d8bbaa9fe50fe1eadbce20","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"026cc77bd0118d0b0fb23c12ea4070b3adedb4901049309b0fbb5d343f23ef2d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0405121a11841816fbb5c37ec640295b707ead5e5e90afceec8c6d26d36b89f6","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"07446d7429225b418915a4cdd3209ca077c65a04e036a316910ed5111aae4483","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0790a6807f6fe23e4136d011f5d24b06e2e93d59f31b54f7ca8c074187cb14bc","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0ffcd91712b8f7a0d5a6255d91431bac699403350a8dd152179a72a5407bd32e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1910d4ec68d7cdf87abad0e5482daa2725be756f3977ce0ec278b06eee95b9a2","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1ac6ce17bff60c7ab63cc5a0613de3bc95d0c885698dcb250d6e706d723721dc","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1edfbe3cd4abbbfa5bb64fbfda2b694dc27bdc2bf43485ed68fa4e009a41483b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20432c14ae8e2b382a936363590051ecced86fa2c7fc4fb3a142a6754ee3a8a1","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20c57c7f4f930cf7d2b679efc13ba7fbecf7f7907701b7050bc117fc055c41cd","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20dbbaa502a201960644c795ec041eda3e1e7f6b22ea6aecfb61d871d82f1f9a","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"22b87309be06b2c25c302beeb768905cf2e4feb40581075301f30e8175af4989","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"22dc20d764c053159f3d9bf81cb2e4491054266ca29cdfed267637828ffc5585","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"230b8c9bddef50a3a05a1f3de54a900d3b7d6faeb31633cda91c740dc4eb3df0","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"238297d169108c98a2d4626af8f21b19c4b6c7cf59dba7f3ee8ebc66431a7b58","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"354b1ed0687723f95732e53506d76b8935f43c4c5a3fc5d2ea8b6e032f27037a","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3795303d12ea1c3511d89431ccc5b137f0422812c4d6391c3dda69ba83bc5138","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3a03fcc14b33be029bf465926fbe4e3b50d9cbbd257c1329c1a09d6e3d51ce07","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3aa6f285ea3287c6e6b6efd85ec823aac3e8bb57b14ed667143e706c263cd02b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3eaaced785877507fe8e3ee6d7b20f5a9c0f3c1c9c3e1008644f14d126f02499","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3fda7ea2b1167a41d35a76af005fe2e83fef0dd3e1f92adb38162bc326aa3e9f","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"4fdcf232f1620222aa8ebb4140f8ff9e4df4e5456bf486b7e9d595ad584bd299","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"55584a5d68238074127db736683181e52fad1bebe6e5c47227f063f1867bbcab","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"65da7551f02a1b04092f4424c3ac603b9678ab2e11d3a2eee871ff91691dc340","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"6ec94cb1c82c8e7d53280737a5ac81a87a2c3700c6756fe2e48135d340e23a34","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"7026d38b22f3f07287af1077384e1cabd2b9c060a726eced072baed3e746e2cc","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"738f282ac1b9e19589766dbd04203e8a353f699d5e20fd2a33231a5a43b2c55f","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"7443e6c90205f089a652d39f2c734b065ee2d31caae75c65c39849fa915121ca","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"794b65f9275a335fe93990764529043569ad0192f9cd5210127d164be6a02d17","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"7accfc45cea7730bbf56250febf86ba494e9aa48ac91ee6b6fdbe18c87125948","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"81f51f0b76d4d4a028609118e73e5d0dea40c40e11818d1448076a2a1459f305","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"82b0b1156dc53f9ba84cacc359e3ff611032fc16c9cb42d40cc22450542aca18","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"85aee33730d076723f6c34d4e6e85fcce12cd9a915dafdd0b0be11a764052b6e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"8940fa1c7065bc2d195ee4dde9f285ef8865512be349e563e8389f270a040c9d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"988a31a8c55bd029da7feb1d59601dae45a62370c0cc71affd91cce4f2c7ad95","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"9c96b70957f19ce12e95ab6707ca9a0ff1dd2b4398316345c774fd3ed009fe0e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a1ad804cbb749acb2715a93cddb0a4a8b005090394f8ba59823fd9abe662b7ad","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a38f6089ac07d5a08077f2a938fa9a9ce3187cf1d0945bf9427cd172c5b0edb7","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a79fbe141ddedfbbb015353053ca0abfdccd6e2f4aac398ef349ab02ea1d637c","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a8b671a92f5e23c45cad8a1b8d273eb90f482db780e90e20fc073db8415882de","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ba10d5c256fa54a16211fadfae959953b61915e76bc9c818bdf27f141f48620d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"bb913418bcd18315762f2b7a90d5b2b627b3a803a94e0dbaf404fc37cf3c0e1e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c14a5bf416c111050f3c01f9eb312019d79dd48c33021d98a6a17be90bc94e15","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c176251ebeb4dd6ed7452613b62ca913760a464b5df07fbd38261c64a43a7f80","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c546460511702c57dce3c2a502e72728ac52f4e17ad01db4e2f1df9d85749433","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c9678a1d0afeb8bebc63313f1948bddf6056aa3f46f60242bdb863e10177e365","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c9f831179ed94cf1dd3e33261efdae1890b83b92efba1fc3ab44d96ff0e83293","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ca8bddcb42c1ad62da6b28232b746387c70bc73f83c9bdaa522ac32aacbf5caf","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"cd9069a841fbe98bb39c002f92e470bd15b4f2fa478338c86af1fbadc560b0bf","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d0252d9787407ca82fc396a0e348b68f2ca460451e133e365d6b87728fae2ab5","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d2ef697ff7ce93a99a1ecb9bada994ad8774fef3f0ebcf089cd9c05331c620b3","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d31ea749491460d333cd226c9b483a00c486c27b079faa606afe6b5fa0405642","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d92081da24f98bc6a986f0b560c3a578136d3424cc67dd149b67bcb0e1f29c21","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"dd98cd647e0fdcb1b2cdd255f2e7d94c26fa423165e9e3d0f2fec25bda4ee4ac","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"dee6843556c75cd55d8ca44339be5632c421daa76b7b87f6c9fbe6f53019f4c1","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e595a69b656f17d5869284ea95b799429d30cb1a24af721d8a7967774dccd0ab","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e6c1290ebdde46d8193a6106ebf8e29a72b8f152685a514a81e64984610d0f59","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e7b8feaf48ce77a449a907bda71e076340c6dd93920023c8fac9eac9fdba0998","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e94cb2a361ab53f63cc6f93fd14b4a8b8bea7eae446cf32e755322e3fc12b81a","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"eab85837f796099b224575bfadd827e056270cf5a6a80ecba8184ab0e5932c88","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ed7e4f1ec742ff5b11ff9aaa80b8d076cb2a6a3976dfcadaffa293b39a5401f6","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ee6d73e71b3f4b0cd58bf4d5471102d6c9f5b892d3bc0a601029200dc711809b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"f01c77ea3e86ff9d0036332decf0cfe00a418350308feb18e16b12cedcbe2c46","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"f097960ef5ad5d2eda9874df327ff0fda3681cf5e790bf2effa00f0b0b3e8894","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"f5c4e42156185abc5aa27dd8331da5ae9c89eadaee3ef25f80a456bf5d31ee46","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"fd895cfafade17d9ab781eaf19976211ff70e49ab529d8e09ae289d6427dffe1","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":66,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":66,"after_last_push_count":0,"trend_counts":[0,0,0,66,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d7c60a347e0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-wh6shv999rvk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n      \"name\": \"d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T18:02:10.15141187Z\",\n      \"endTime\": \"2026-10-09T18:31:17.847497361Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d7c60a347e0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-wh6shv999rvk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n      \"name\": \"d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T18:02:10.15141187Z\",\n      \"endTime\": \"2026-10-09T18:31:17.847497361Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3aa3a58fd700\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r9mpb5rglfkx\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n      \"name\": \"501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T18:02:09.983214688Z\",\n      \"endTime\": \"2026-10-09T18:31:17.777246897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3aa3a58fd700\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r9mpb5rglfkx\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n      \"name\": \"501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T18:02:09.983214688Z\",\n      \"endTime\": \"2026-10-09T18:31:17.777246897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f8cfca18160\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-d9vjk88cdxmv\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n      \"name\": \"715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T18:02:40.105550226Z\",\n      \"endTime\": \"2026-10-09T18:31:47.946839601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3f8cfca18160\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-d9vjk88cdxmv\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n      \"name\": \"715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T18:02:40.105550226Z\",\n      \"endTime\": \"2026-10-09T18:31:47.946839601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"01c7c9aba88a6bec22099f9efa23436203147ffbb00db28596435d46a08f8a58","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"0d88e33eff3b08810a358f00c8d58a2d311131200ec6c8ef38f667c486c55827","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"101829a1e4b673aa4d7ffb71716277ad9438dad959bbce3ad102021a727eb3fe","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"13de764af8574947f7d4d0670b47cfd74c8f2223eef88b15d28b542d71598982","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"18cbcc3172a62339984a3d37ac2466dcb61cbf74717c9d34584498668e6449e7","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"1a225498f788e362cfb0798ecc880ee9e0d615e421befffa8b4c39c1db09ea76","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"23541533d46875257de182f86b81eb800fae0069577baf283b2393e3ecd99d6d","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"270f52452b182f15f724ad102109f03fa2af29e0483c091f90e700cd80a62b63","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"2ae8b16399c641a572db2f6a75461482166a2cd3b4fbba6fc168f57bfa5cb619","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"2febc8fb20e64b0398ded29dc23013a05f1468e452028bf82ed4f95823d3e067","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"344bce9d5bb6db55aa99b4d0dae42ea14b220a1bbda168e54c277877dd20c8ba","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"36bc2df6248e6d14334a51d93f771ce0e45399416a0e41167b90f40cac1dccd6","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"3a3d2162b324369cc9d26692d4078318e79e4e939c2e0cd7569a424308ab189d","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"3dd4cf855356a60f412d65e778d1d59b4a3d625c3e48837dad30fa890f5c5ea7","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"42e03a759fcb36e1ff575851b5b1f1da7f4bd1ebde306828e827dc02e2199766","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"471888eab8c388e835c310bd537c7afbb02ea604ccd87bd17c259c2e24db7e12","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"47c55e31574be527ea313db7769d85b3ccf1d32158e37662d651889d8dce39cf","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"4bdea9cdd7191534eb3f205fc41751072adef3d93c0e8cb4a9f543ddbddc8724","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"5326b37b127a327cc885a7d06d11d2e5d1ae127ff662425651996afa5995d4c8","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"573c2968f03b5e1da4c26a98373bd1ec66da67dde8c5072bba353f3bec23cb8f","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"5d43281f44963b75ff8872a084fd7611e5d520a8395c99ac66916a5f44309d46","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"5fc73119cc4809fd8d58b3d22fa26a7089966e8c78623d913341e28668630c4d","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"605577627fca4f1f1e9afdeea753a9b0f1014b2292b57c1e35fc2434b6a85524","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"648b2e6e6b01fe5aab9f8c1ce0cff5dbcd59c3427b11e6628df8162058113cea","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"64a208342c453fb0e3e24af80d2b52ba48d347f00960fd0fb86861c86009e388","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"697453e7f696f54f41edcb34030e48efa337b5e8853e8f54ed448226cf012cf3","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"6acc3aa5ad77e5f9b1c9ed7b19fd706b537dc9966ff8a237c8745e71ed06cb17","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"6b365936ba6c5dc6e4955ca7b030bb08117bd6ac40551608167f9a3365250583","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"6bd0886fe9be28c4793f4cbd79418d8f356d5615ce582d8d0371e979a9c5f77e","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"6c5019cbf95e4513f1c0325e65420dcf50b2b8b9b1bc354a5bc737088dcdb0dd","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"725401be84daab8b061390e9b75a1844bbed2e06bc3458add8fd0998fa58edf6","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"7834d516ec38d9112f9510dec32a0654c9a7a541797540701f62a268ef5c8e78","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"7a09fd0d26a1837bd2f82a8277d0513b55de1cfb905ad76289110b9aa7c6097b","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"7e936789cef771ddd9316ff31b81612b15375d9f6e0b7e2d85d82bd80d1f4752","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"80d08043ecfaf539ff53744c99fc856a641ca23589f93616ad5fd6281e5d2505","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"823aa7b3881e8fb041db03af06d9fa85a03a6954f019b4555aaf30c8412e5a43","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"86d0639f8a612df296007f755fdbe489736d2e4139e62ca248d9792696071e0c","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"86e6e152f41be537ce62e007989121fe49a0689d42902ba0575419da7d1cae09","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"8be7e936168bcef852e361261d8601b6e2bd9c2e2e5761763283bfb55577d5af","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"929e070060761e0145c7786b83b10bf8854e5806952eb8de75267c6508fd7347","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"95092931f0b18c876d556c51bf269aaa70f521731d5e8948e3036ee0f211a9b5","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"9522ddba404ef96e63c679a657e30578373adb9cf594a75efc0375d22fc4ed5c","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"99108240d9af947608575ec6e131c4e8ea683becdcb1c6901189bec0b24500ba","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"9a69793d927870af610e2db8ebc5fb307ca450c095c5a0f6f70d0ba353868781","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"9ede499c304c2151498d5fa3f541dd8841118ad3c2e27507275b36b37bd4fa10","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"a51a5142b300cc25f83121d87531a6477ee39ee81964278209d741153cbef2ce","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"aa7ef0a640cd2e5f193ec1087698a362ab938fd455bc16df8d04503eb0313d89","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"ac9d571735b7d2d819291b1cce9460a34edeb058e38bb2703a76fa82e279cdc1","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"af974a64f9672e06da3b97d10120baa2a2f2a5d5c7ea0999bd68dac78aaa5fac","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"b45da6a81fea23d6421698db5bd755ac99c4569b5c386a8df2278b531b67b4fd","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"b7b1480a1ea53486737434398aa7a46d927a9d5857abd693d1a00c9d13054a37","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"bf0240e034eb5e7bef668792af253798232bc78025b0cfe8329b245ab54aff3e","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"c9a888ba363bac1a8695dd65eb2abf121932544cc844047c0c9256e6cecb2493","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"cafba5839e877254327e3080d52936c7a82ce89903ec574f9060c3dec19d8fcc","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"ccbadab87d718693551dd8538c82af6ce15efeceeac75de00a658396f4727bcf","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"cd9e8e56479966f53ef3434d48eb0be4e3f969ddd83de89e192a9d68b26a3eae","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"cff19b5d202fc005cd43e8cdf242df5b3a41239c4346b5edd14d96775c35cc0e","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"d24434ab2966d55d5744a0237f93d89b9f15a01822488b010acc27aa17508832","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"d41480da135c16a57e520650419cfa359a9141d8dfe06e4e1787793d738cc6fe","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"d8e1a5f9d52ad8aee666439da2a8fa3c13b37b36cd8b70e932dec4d49375ed45","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"d94749aff98696c868c3dd015c137aa69ded1b9c88ae9994b2fbdea0a163c668","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"e63114a71a857978b52cbd146d9854d7e3d351cfde3aff02e044cadc2bdd2d58","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"ec4b90a129599d5500cd5af07d3433c687996c1b739d2913a2675d196f5b546e","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"eff3312fe070323cd5940b3c5e218a49e22dee4584f1fa0e3685bd035e905815","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"f266521e812d721d55035420d232767ea4a41f7eb1df70515690979a7dfbe6da","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"fe5361e6f948427f64311e91b3e4d028f213c3abcab12243ef538ed838350160","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":62,"runs_affected":62,"failed_runs":62,"run_impact_percent":19.935691318327976,"anchor_occurrences":62,"after_last_push_count":61,"trend_counts":[15,29,19,4,0,2,8,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":205,"prior_last_seen_at":"2026-10-11T17:11:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":62}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T20:59:27Z\n  Ended:       2026-10-12T21:19:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6043392-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T16:22:14Z\n  Ended:       2026-10-12T16:44:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9626368-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T14:00:36Z\n  Ended:       2026-10-12T14:17:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4606592-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T14:00:36Z\n  Ended:       2026-10-12T14:17:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4606592-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109622919087788032","occurred_at":"2026-10-12T12:31:13Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109612109212946432","occurred_at":"2026-10-12T11:48:16Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7467,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109555123024826368","occurred_at":"2026-10-12T08:00:48Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109331063628107776","occurred_at":"2026-10-11T17:11:29Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109291724151984128","occurred_at":"2026-10-11T14:35:05Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648873588166656","occurred_at":"2026-10-09T20:00:42Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108569711095582720","occurred_at":"2026-10-09T14:46:09Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7444,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","occurred_at":"2026-10-09T07:02:15Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","occurred_at":"2026-10-08T22:02:45Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","occurred_at":"2026-10-08T18:00:34Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","occurred_at":"2026-10-08T18:00:25Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108163022403932160","occurred_at":"2026-10-08T11:50:06Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7401,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","occurred_at":"2026-10-08T11:09:46Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","occurred_at":"2026-10-08T09:41:34Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108128762271895552","occurred_at":"2026-10-08T09:33:58Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108123255498870784","occurred_at":"2026-10-08T09:12:04Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7420,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","occurred_at":"2026-10-08T03:54:28Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7177,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","occurred_at":"2026-10-08T03:07:29Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108015628659986432","occurred_at":"2026-10-08T02:04:25Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7417/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107967978476670976","occurred_at":"2026-10-07T22:55:04Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7417,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107964840596213760","occurred_at":"2026-10-07T22:42:36Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107926490833227776","occurred_at":"2026-10-07T20:10:13Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107926353130033152","occurred_at":"2026-10-07T20:09:40Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7305,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916465603612672","occurred_at":"2026-10-07T19:30:23Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107914072107257856","occurred_at":"2026-10-07T19:20:52Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"bb0b7baeaee3903b2c32b416b3b8a2cd08cc9041556204906cc5c3c0dbf4e3a4","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"bb0b7baeaee3903b2c32b416b3b8a2cd08cc9041556204906cc5c3c0dbf4e3a4","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7322/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875761300967424","occurred_at":"2026-10-07T16:48:38Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7322,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107872947002675200","occurred_at":"2026-10-07T16:37:27Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7410/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107844183950626816","occurred_at":"2026-10-07T14:42:46Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7410,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"f0fdfb1a7d35edcb5a2bf5e1776a6f8480ed63182bae2c515bbe819b35402c27","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"bb0b7baeaee3903b2c32b416b3b8a2cd08cc9041556204906cc5c3c0dbf4e3a4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107802095104036864","occurred_at":"2026-10-07T11:55:55Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107778619966230528","occurred_at":"2026-10-07T10:22:38Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107752436901875712","occurred_at":"2026-10-07T08:38:35Z","signature_id":"7a43480e65ff97585b06bb990026d9f475bc404b7071e50eb5d7c926e16db6df","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751899905134592","occurred_at":"2026-10-07T08:36:27Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7403,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750854802018304","occurred_at":"2026-10-07T08:32:18Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750168211230720","occurred_at":"2026-10-07T08:29:34Z","signature_id":"33026751834bf5223e46b5e0fa41f4843e6b143bf0d32bcaa237cc03345d8d5c","pr_number":7370,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750153682161664","occurred_at":"2026-10-07T08:29:31Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107680144293367808","occurred_at":"2026-10-07T03:51:19Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7097,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":42,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":42,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,42,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-29T01:37:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x212fbc084a0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-8hkmlh8qnx7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\",\n      \"name\": \"a5d399ee-b335-46b2-ac26-ebca4137d374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:04:53.824928754Z\",\n      \"endTime\": \"2026-10-12T16:34:00.204962099Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n            correlation ID: 2d28ca59-634e-45e7-aa28-3231a3cd431f\n            client request ID: 9ee94869-89d4-4850-9db2-977e7eaffbfa\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x212fbc084a0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-8hkmlh8qnx7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\",\n      \"name\": \"a5d399ee-b335-46b2-ac26-ebca4137d374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:04:53.824928754Z\",\n      \"endTime\": \"2026-10-12T16:34:00.204962099Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n            correlation ID: 2d28ca59-634e-45e7-aa28-3231a3cd431f\n            client request ID: 9ee94869-89d4-4850-9db2-977e7eaffbfa\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d33bff38b20\u003e: \n    failed to create HCP cluster ea-listhxx92t: failed waiting for cluster=\"ea-listhxx92t\" in resourcegroup=\"ea-list-6s5nhp2lwjz2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n      \"name\": \"b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:05:25.793338411Z\",\n      \"endTime\": \"2026-10-12T16:34:29.312060412Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n            correlation ID: c3bdb4b5-d9e1-4722-8d4b-0801c32c52a6\n            client request ID: 6f856d28-2856-4395-afc1-9135fdfb41b9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d33bff38b20\u003e: \n    failed to create HCP cluster ea-listhxx92t: failed waiting for cluster=\"ea-listhxx92t\" in resourcegroup=\"ea-list-6s5nhp2lwjz2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n      \"name\": \"b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:05:25.793338411Z\",\n      \"endTime\": \"2026-10-12T16:34:29.312060412Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n            correlation ID: c3bdb4b5-d9e1-4722-8d4b-0801c32c52a6\n            client request ID: 6f856d28-2856-4395-afc1-9135fdfb41b9\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3390eb576ac0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pj6qdmqzbvvk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\",\n      \"name\": \"9ca81053-a842-4371-b45d-1725686e5689\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:04:50.549768464Z\",\n      \"endTime\": \"2026-10-12T16:34:00.22620878Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n            correlation ID: bd5b8009-857d-40ed-a78e-2ded6716c304\n            client request ID: bd2fbaf8-a68b-4aff-9d2f-f237a37aad2f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3390eb576ac0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pj6qdmqzbvvk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\",\n      \"name\": \"9ca81053-a842-4371-b45d-1725686e5689\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:04:50.549768464Z\",\n      \"endTime\": \"2026-10-12T16:34:00.22620878Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n            correlation ID: bd5b8009-857d-40ed-a78e-2ded6716c304\n            client request ID: bd2fbaf8-a68b-4aff-9d2f-f237a37aad2f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"055f9a9b6c244ec70223014863e02389b9a479621a6d56cd84fc1a590883325e","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"06d984b5bd61e16ead3a82aa893ca35dc9577f417c04093909481769e3f573b8","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"0a7b025a664cd937a27c1586ab0853b5c1ef3166363dadc1edebbd26f45f0f59","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"160071fa1d66581da10fab1353ab09d80ff5734975070cab9434ca0e1322ff49","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"1ec3f7dc73b39cc393755b265af0c51797463b27d90c21e0a2834ab16544dbf0","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"40e270b1250c572d95ad297676751fd03a52912e32aa17a878d7239f4fb1933c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"434f1f50bc3fec5a453eb5fc6ce070a5ad6cf40f73119f0ef622c86873aa56ea","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"44ded8bb1a8d8a2ac19d80e715a411a4b8efc7b7cc9f887538780c1e34b93ab9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"47dea9fa6d473ca88a0d6daa3a8a61ce6d04ebe38764f2f66d84f10a437111c6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"4d85fc8c6214af114002c843ebdc86a94cf3126e2e1355356d3ebd73640aa90f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"51071ac504e6050a1fd884cd82764171a1d38ab7411e9ae858f536e87567f055","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"586593e2fff2e69d645fc42dff24f41310743ceba1deccc3b657c96749c0928b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5bc4f6a77655afb07a6100dcfe611731a929b63429a3f3272baa0c1801222753","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5c7543311511f19fcd39fd25b976d88fc18cf2a06f89b8cd4da4fdcb65f57861","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5f344ec756d50afef29819a6882bec64ddbd8b7b94b117d33badf95b4228e5c9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"61789678564be0aad54c9098dc019c63c0a07abf6db9351123669661958b90b1","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6b095616bded2736b44fd15719c1d4b68c8ddb854dc8491ee2f30e52888a0be7","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6e1b7db0d8aeab353439c24ad4514730ee14c5e2353bca21858fe1f9675242cc","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"7c6abafe393255f1739fdb387bd94e187d718daaaf8baf7ec69854b4ddf141b2","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"7cb425f2d7951985395b91345c374be4c0dca4d08831fcfb07f410b4baec6807","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"8181f041b308c072f1833b4eb56517c581e5c8e4e9cd0a8ea0e6808fccc7a57f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"844afb351725b6475ddd327ef78fbf5da706650b28665d4a6e378e60144b8e25","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"85acf0e937ffe0027b15bd44eca4c7c42194560ce1678ed387a13da6d7aadcb6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"945c8b8ba229efce78c82c55117d9808b09ecb3e3f5b4cd8cda0f81bc283543f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"97fe8f81d50ab0c04cc31936087a1cb6ccf4abd2ff0897e188700ec58b85890f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9b8bdf9e25347ed8bbd86e38e47f8af0335efec4efa7ea5222d2c6561d3fadfb","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9f01d929e13b36781c18a8d7f12e2b1fcb9913e0a3fcb25d7d5ff885c820a357","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9f58aa2dc15f7e7336a495843cc0f699c3e5b143f75bf07f3842358d658bcdca","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"a0f1bf26985a99e6539af709b7c6cfaeed3b89c57f24089e9a344841e1966c77","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"a312b9a0ce9a609eef608f7ee363a1a717dc275fe1d0ca99a6b39591f5f7c10b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b2b126547a0bcacd7f8d6fadea91d4a58f4981406c524e6014bd0b7da6c92d4d","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b7d3cff9604baa7adb7447f8feddb9b4e1f30933ea4005d1bf550006ccbe8684","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b83badcc40496e121c06a8423ede3d7aeada8dd0924a4287300c02e0fa12d3e6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"c7ae04b38766210a8108f94bb04d957dfc5bca36ff0aa8754d485ec2530ed6a3","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d1805d4cf77820ef4f29d07237f0247a441e0228a2174ebcae5ceacb2de61aa9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d859241facd880cf6291230bc4ecdb749dce2456ddc8c4a192721ae181952d05","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"da295e3f891ebd53e53b834c83d5f2d2d8c7990e91838251fdcf5881a2a1f90a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"dbf86b79040a8aaf8dd94a9133ea7c86d12cf15986aa135f1c6a41485545be9c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e642f26ea0a98605c1da7d415345036ed06aed0f4668bc7504474fbfa4f815a5","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e76fe38f94cc0c24f57a2b14af3523d40d7b5c15cd0e3fefac98dc8b85b99bed","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"ed62ccd125102b89fe1e6cd6a42a65c596b410194665770b4c1ecb63e1ef3a7a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"ed6cd23384ef866a46342ced82dc3a93106a5447a643b0459d4e1595765f54a8","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":24,"runs_affected":24,"failed_runs":24,"run_impact_percent":7.717041800643087,"anchor_occurrences":24,"after_last_push_count":28,"trend_counts":[6,9,3,2,1,4,5,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":167,"prior_last_seen_at":"2026-10-11T23:29:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":24}],"full_error_samples":["alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T20:51:04Z\n  Ended:       2026-10-12T20:57:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6043392-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdfeve1hnh3ip3571ig9kpg8ogs5h0t-r4w0d6n8e5u2v7j\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tdfeve1hnh3ip3571ig9kpg8ogs5h0t-r4w0d6n8e5u2v7j/etcd-2 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T16:16:01Z\n  Ended:       2026-10-12T16:22:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0290944-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdbebgjal25ks2eu7kru50siu2d9fk7-d5m6i4f8v4g8r0f\", pod=\"kube-controller-manager-655fcf6674-rv7x7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tdbebgjal25ks2eu7kru50siu2d9fk7-d5m6i4f8v4g8r0f/kube-controller-manager-655fcf6674-rv7x7 has been in a non-ready state for longer than 5 minutes.","alert fired 20 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T13:06:43Z\n  Ended:       2026-10-12T13:30:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o\", pod=\"azure-disk-csi-driver-controller-67b988dbd6-9pjgh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o/azure-disk-csi-driver-controller-67b988dbd6-9pjgh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T13:06:43Z\n  Ended:       2026-10-12T13:25:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d\", pod=\"multus-admission-controller-86cf4f5c69-mqbpc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d/multus-admission-controller-86cf4f5c69-mqbpc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-12T13:06:43Z\n  Ended:       2026-10-12T13:25:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d\", pod=\"multus-admission-controller-86cf4f5c69-jhk97\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d/multus-admission-controller-86cf4f5c69-jhk97 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-12T13:06:43Z\n  Ended:       2026-10-12T13:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o\", pod=\"azure-disk-csi-driver-controller-67b988dbd6-b9n46\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o/azure-disk-csi-driver-controller-67b988dbd6-b9n46 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-12T13:06:43Z\n  Ended:       2026-10-12T13:34:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o\", pod=\"azure-disk-csi-driver-controller-67b988dbd6-cg97m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hjpvhu8glv53d47p2282741ri22c-k5r0i5u0f1o9r0o/azure-disk-csi-driver-controller-67b988dbd6-cg97m has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-12T13:06:43Z\n  Ended:       2026-10-12T13:25:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d\", pod=\"multus-admission-controller-86cf4f5c69-kpf7b\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hhrh2u47723p4jk6svvgmgqch9ah-a3w0n4n2x3d1l7d/multus-admission-controller-86cf4f5c69-kpf7b has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-12T13:13:45Z\n  Ended:       2026-10-12T13:24:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i\", pod=\"openshift-apiserver-6b4ff7ff95-9kn96\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i/openshift-apiserver-6b4ff7ff95-9kn96 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-12T13:15:42Z\n  Ended:       2026-10-12T13:36:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-7cvh9\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-7cvh9 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-12T13:15:42Z\n  Ended:       2026-10-12T13:36:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-8mskw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-8mskw has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-12T13:15:42Z\n  Ended:       2026-10-12T13:36:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-j42hz\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-j42hz has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-12T13:15:42Z\n  Ended:       2026-10-12T13:36:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-mkdz4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-mkdz4 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-12T13:15:42Z\n  Ended:       2026-10-12T13:36:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-5xwvh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-5xwvh has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-npkzv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-npkzv has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-m8dx4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-m8dx4 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-pnr6f\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-pnr6f has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-gw6tj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-gw6tj has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-5x6tc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-5x6tc has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-6lcvg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-6lcvg has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:24:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i\", pod=\"openshift-apiserver-6b4ff7ff95-mksfs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hdubi1998hba1cbuuv9q41lduclv-j3t7o4c6b6y3x7i/openshift-apiserver-6b4ff7ff95-mksfs has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-12T13:15:43Z\n  Ended:       2026-10-12T13:36:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j\", pod=\"multus-admission-controller-66868c485c-bx6hr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2td8hir7bcqf7v5h045nvaaq6p25d2at-n3r7k8t0l9m8z2j/multus-admission-controller-66868c485c-bx6hr has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"4e6814427594ada40105d0c851f37c68ce6084d56587c92bb86a0a5911644b78","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","signature_id":"36ccc2be84055641dbb6036217141bc4c104ca4b184f2aa2e64ec66e411c6dbe","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"4885c3029699261783a90f8a5a1cf25063711e5d2be18ed2e9aec6344e30e22d","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"eb4f11f09029df76c1cd0c8884283989b59eee33dde5e273ada7dc186fddbab8","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109519598771507200","occurred_at":"2026-10-12T05:40:39Z","signature_id":"6f8e2d109f354a776a1acb2c7fa60149b6a8d7d0329aa7b37c469a8320c9fe0f","pr_number":7430,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","signature_id":"f68370508ff6caa5c18b35640c5e7236383094944a4be830b451b52b18add7bd","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","signature_id":"a3c7d3032023cb3482ce8c92f0dd2a976d734609da54cda79a8760b5940a0595","pr_number":7424,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","signature_id":"24628989d0288208601920b5e851ec201bb5169aacd7861a259ad071fc8838f4","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6452/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109140137769177088","occurred_at":"2026-10-11T04:32:49Z","signature_id":"6a61843b5445b12ff0d8e5e34d69f5aab37878cf077a6ff5a1fe09ff25a81ff5","pr_number":6452,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108819747389837312","occurred_at":"2026-10-10T07:19:42Z","signature_id":"c86eaf93a1267e5b8897c354444574607bb8dab7fce27496ea0d79db2502e9d3","pr_number":7374,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694594228064256","occurred_at":"2026-10-09T23:02:23Z","signature_id":"45c56709fb0815fc5125e96d111279ae3d1aed3d93168dc727565ba057fd9a0f","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108564560276361216","occurred_at":"2026-10-09T14:25:37Z","signature_id":"ef06df0b30d9f19d0a26d641f64c4b995322c1aeb343d1676cc66991c1e87183","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","occurred_at":"2026-10-08T18:00:26Z","signature_id":"1e60a482d86ac260f86e568bf243b409c0a25d681cfd32fd6d4ce13302592f5a","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"dea7649e1e363ea4afbe251966c45c45a904d878aa58e50d8b09b9fb6f157106","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","occurred_at":"2026-10-08T03:07:29Z","signature_id":"33a563d1f9854fc5c2d805dce229ec37f3683cf9f20862a546be44d8addf4f52","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107926490833227776","occurred_at":"2026-10-07T20:10:13Z","signature_id":"74818e91a0caa04c1765b38bbad74a23ca21673b3ee1229ea2981f98f72f1e52","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107908786537631744","occurred_at":"2026-10-07T18:59:52Z","signature_id":"156501ac5602097c0938c316b8717eec9df92b585d0d3f3819bbc868a6086146","pr_number":7399,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"1c9a57ae155d22074516444f9bdd42c3eb77f6c4185d2704e7d5ec64885ce837","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7411/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778164936704","occurred_at":"2026-10-07T17:04:36Z","signature_id":"e58cbbeedba7967d85e44f7a9a6e4300651d973f923a375d1899466c0cf4ac3e","pr_number":7411,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7410/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107844183950626816","occurred_at":"2026-10-07T14:42:46Z","signature_id":"260ad305c6a9edb3a0d9ff7ae59c6ed719a99dd03de1fa7d3c3022010b26b23a","pr_number":7410,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"5099bcc506260d26581bf59530494d1a9f6bc90b5796a9d216da2138646a67c4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"ece3af58d3fb6cba54ec581a6635f6e1e2add57872da32b2115f2e616adaf1af","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107717187903950848","occurred_at":"2026-10-07T06:18:31Z","signature_id":"753a35995836a60d7340477581bda7ccd89cec93b1ccc0f139677a210c813f54","pr_number":7402,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"5211917d4010780a3db4438b5c6318344f5122605ba573b6a6b2cc8edda6c262","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":23,"runs_affected":10,"failed_runs":10,"run_impact_percent":3.215434083601286,"anchor_occurrences":23,"after_last_push_count":49,"trend_counts":[57,23,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":79,"prior_last_seen_at":"2026-10-07T17:18:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c574774040\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-m842v74xwlhh\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"name\": \"310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:10:29.82492819Z\",\n      \"endTime\": \"2026-10-07T18:39:31.852478192Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-2tvj4-977r6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-977r6 to exist\\nMachine v20261001-np-1-2tvj4-xn4zw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-xn4zw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-2tvj4-977r6: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-m842v74xwlhh--managed/v20261001-np-1-2tvj4-977r6-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-2tvj4-977r6-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c574774040\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-m842v74xwlhh\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"name\": \"310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:10:29.82492819Z\",\n      \"endTime\": \"2026-10-07T18:39:31.852478192Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-2tvj4-977r6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-977r6 to exist\\nMachine v20261001-np-1-2tvj4-xn4zw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-xn4zw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-2tvj4-977r6: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-m842v74xwlhh--managed/v20261001-np-1-2tvj4-977r6-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-2tvj4-977r6-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d8b7da68c20\u003e: \n    failed to create NodePool np-74nbzb: failed waiting for nodepool=\"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\" in resourcegroup=\"rg-candidate-4-23-z987ndt4v67c\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"name\": \"ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:12:58.862762914Z\",\n      \"endTime\": \"2026-10-07T18:41:59.529725798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97 to exist\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Network/networkInterfaces/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '2' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d8b7da68c20\u003e: \n    failed to create NodePool np-74nbzb: failed waiting for nodepool=\"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\" in resourcegroup=\"rg-candidate-4-23-z987ndt4v67c\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"name\": \"ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:12:58.862762914Z\",\n      \"endTime\": \"2026-10-07T18:41:59.529725798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97 to exist\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Network/networkInterfaces/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '2' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2af3a314160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-gvbd6v968qsn\" to finish creating: GET https://rp.j8893056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"name\": \"0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:57:49.24099524Z\",\n      \"endTime\": \"2026-10-07T18:26:50.588266562Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-vh9lt-cmt4b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-cmt4b to exist\\nMachine v20261001-np-1-vh9lt-j5d2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-j5d2x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-vh9lt-cmt4b: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-gvbd6v968qsn--managed/v20261001-np-1-vh9lt-cmt4b-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-vh9lt-cmt4b-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2af3a314160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-gvbd6v968qsn\" to finish creating: GET https://rp.j8893056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"name\": \"0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:57:49.24099524Z\",\n      \"endTime\": \"2026-10-07T18:26:50.588266562Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-vh9lt-cmt4b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-cmt4b to exist\\nMachine v20261001-np-1-vh9lt-j5d2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-j5d2x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-vh9lt-cmt4b: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-gvbd6v968qsn--managed/v20261001-np-1-vh9lt-cmt4b-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-vh9lt-cmt4b-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"f3c9ccebcdb83b8eb8b3305c00393328a72b529c5a2452faaa135e95a8164391","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"3111f51ae03691646571b56f93f83f1a4aa75f6307d6e356732ae922fa08cb09","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"3823b66b09bfd6e9e4ad92f72a9d4f3ee3c224b37b1c8f1f8ebda27b147d7c46","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"7649741d4e6001a4a976b6c337ded5372079a83044d0f0df24c3502c5d934dc4","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"94460068fa46dffaa7a997415d9e364ab28771906e158a3e989a36795e455e7d","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"f5aa64a25fb502f45efe4709b5d67fc37312041962fb3ca94ee6c6d91283814a","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"87b8ed371a8aaa18808fd97c59919f61f332deb12ffa10d0bd4e46b72315a96e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"0e2781c957524dd582ad9d58b73f8540584537b1abf21b9ba0da2dd49354d02d","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"520474359be0f062cf8b98378dd4cc9e921db0bf50d302b2dd3eb02b1d62e274","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"e523689aea35fd7e6b880b1921b71e1d38fcc318168540029872f15f62fe2771","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"2522196ffab484807654ebdc0cf7d0567bc80e6f3f38b2482ddb3bceb953bbe5","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"33f522deb518c9faff818f3ebc3464e66c1d30a6e753a3a196af8f13e31f4624","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"7d1cd6af866cf990e6b26ab81fc3a9346742d73a2e9757eb55f2c7565d966db4","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"128685f67ffac53964edbe30eb85e252710a82d7325bb00e9be19d64a0dfb550","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"bdc85c73bc1b93540460da38830900bf54f131c55a8128f1abd00806c4591690","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"0e5eed768e639c704016ff6d0901d55fa558c49fd15dda10ad7f2477cc4d65c3","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"44d25dd92f2c1f9cc76ea6b88a4f23c16c0ebb713bd0c5d55d9a1d08edf04059","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"99b003f7a7a441d509d8bef27060bc681040db112029ece09584795fb10dadfd","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"41f661f65c04dcc379b400b33f94f5db1cca5ca24f1ddb4db781f3990be4d132","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"734ef25191ec5fa5c8dcbe8786f5f56d52fcdf12b3ec9ca735b1f1776ac9e851","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"0de4e74aa7302328587b11276c0a2114131a5a3b2f9d1227450eae431fac0c74","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"1b3a05a3a5dbb0b03df2aaec8331ce4285abcace77601310a4833563c35a01b4","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"92f959746d1f89cf28cbdaf39ae77041de5bfce5280761a1e7841543df6376fc","pr_number":7371,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":22,"runs_affected":22,"failed_runs":22,"run_impact_percent":7.07395498392283,"anchor_occurrences":22,"after_last_push_count":16,"trend_counts":[7,2,7,6,0,0,7,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":82,"prior_last_seen_at":"2026-10-09T22:55:44Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":22}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T22:07:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nlt8k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T22:07:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nlt8k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T22:06:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2519552-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2519552/providers/microsoft.monitor/accounts/hcps-j2519552\", namespace=\"ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nbzvl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T22:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2519552-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2519552/providers/microsoft.monitor/accounts/hcps-j2519552\", namespace=\"ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nbzvl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tdfrhqfhgs9sdc5d6upkckvedq8cool-r5h5j4f0m7l3f4b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T13:44:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T13:44:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T13:45:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T13:45:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4879872-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.152:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4879872/providers/microsoft.monitor/accounts/hcps-j4879872\", namespace=\"ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-9ncdp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2td8hkede6pik7afuli4rqulo2mdm0ib-ksm-hcp-metrics/cluster-api has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"5cdd689b990f511feccbe071c3661b0d836cd6b71dcc489745470445ffd67711","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109740763322519552","occurred_at":"2026-10-12T20:19:29Z","signature_id":"1f8e213972bab57fbe586fb0bcc701f04d1d69dc2413a26b541db4737b3a7545","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"ace1503ea85ab8f2b4df98ae24ce5d1ca17bf7dd773d4068dfe89225c5ed9594","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"18f5e818c6420e89e3d88cbc04dfa007473088c4fb38a457467fa320874574d7","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"a980a63c43707ea1bb98420a0dc8810e1d7f957a882e906bd2844388f2b4bbc5","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"c2c5cbd8dc969313ed2d41b08a27fef289acf32d3365fe9e6a45d7383b454d11","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","signature_id":"36fc6cf9b7b57226fae3978e4f0d34c96bf304895ef33b2f585810930ebddbca","pr_number":7424,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7448/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108692920105177088","occurred_at":"2026-10-09T22:55:44Z","signature_id":"7de35d5024d3b3010d077617b0ac7fd8f62bcc84b4d231065a5dc6177210187d","pr_number":7448,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108611968167317504","occurred_at":"2026-10-09T17:34:03Z","signature_id":"6fd1487b882a46fa816895ea3781fe05946c86b7aff2a9b326e2ac82c2028920","pr_number":6904,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"ceaff42bdfa032f28d1f49bd3b397444e672e2f501892eadd1f4e851d4b9d1c5","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","signature_id":"95e8facc46b0421aa919d496d5b63d8a35fd8772adf859c61c4dbd4480045aea","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","occurred_at":"2026-10-09T06:02:56Z","signature_id":"f5d956fa0b22d0c0a85159f69504906557f12343cd530a352e6ae14cf54c0d8b","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108409414854643712","occurred_at":"2026-10-09T04:09:10Z","signature_id":"35d2f92aafdf8f1dcc0d45fecd2e1b20a6e9189a9a234f533477e9527641239a","pr_number":7105,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317188833153024","occurred_at":"2026-10-08T22:02:43Z","signature_id":"c904114929467bb32c67ef275024818849dff8d5a227072eff3a386882ff848e","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"4bef60ee24c495084afb20bb4a919adeb209d9fae5a39e972a9d5a8c119a1d43","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","occurred_at":"2026-10-08T18:12:42Z","signature_id":"7e6534085d42dc6118c7a37dc3076a5592621ad332aa0f0dc2d28d01dd263625","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","occurred_at":"2026-10-08T18:00:25Z","signature_id":"481b22a2ae9014aee1f61475b4be429b12de22c8d096e16cf3cf96337a687bf6","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","occurred_at":"2026-10-08T14:06:48Z","signature_id":"b50a31093336d8331ae783c4d9e7666f297d09f74d347cd9133e1c15e42f6f1b","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","signature_id":"2bb5f552ea27ec9022c5035d204ea8f422b813234e438a587a8af153658a6644","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"7e1b30717a7056324794286c92396616dc0d26090203314168033730260fac49","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"10d4a1a04107eb655251aa6f4caa6a71839828780a3edffbdda858f07d1855cc","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"de6beb6bf4334e9b143d9dd4dbc06e85520a4d8bbe278fddc25171d8b71278b4","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":22,"runs_affected":21,"failed_runs":21,"run_impact_percent":6.752411575562701,"anchor_occurrences":22,"after_last_push_count":5,"also_in":["PROD","STG"],"trend_counts":[0,15,1,1,1,1,3,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":113,"prior_last_seen_at":"2026-10-11T12:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":18},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xaa70fc27a80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pcvlks6fdkqz\" to finish creating: GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n      \"name\": \"26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:52:23.241617081Z\",\n      \"endTime\": \"2026-10-12T16:21:32.102166958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-q4ttl-4vcvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-pcvlks6fdkqz--managed/ingress-cert-le-np-1-q4ttl-4vcvt-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-q4ttl-4vcvt-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: RetryableError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"RetryableError\\\",\\n    \\\"message\\\": \\\"A retryable error occurred.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"RetryableErrorDueToTooManyCalls\\\",\\n        \\\"message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX was used to perform too many calls within last 5 minutes. The number of ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n            correlation ID: bfc832f2-b15d-4a34-8c01-edecd304fc1a\n            client request ID: cd0488d8-da8e-467d-8248-7320c5456b91\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:137]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xaa70fc27a80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pcvlks6fdkqz\" to finish creating: GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n      \"name\": \"26ef0fd5-d97a-49a6-8fbf-0d954df5101e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:52:23.241617081Z\",\n      \"endTime\": \"2026-10-12T16:21:32.102166958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-q4ttl-4vcvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-q4ttl-4vcvt: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-pcvlks6fdkqz--managed/ingress-cert-le-np-1-q4ttl-4vcvt-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-pcvlks6fdkqz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-q4ttl-4vcvt-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: RetryableError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"RetryableError\\\",\\n    \\\"message\\\": \\\"A retryable error occurred.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"RetryableErrorDueToTooManyCalls\\\",\\n        \\\"message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX was used to perform too many calls within last 5 minutes. The number of ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2636160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26ef0fd5-d97a-49a6-8fbf-0d954df5101e\n            correlation ID: bfc832f2-b15d-4a34-8c01-edecd304fc1a\n            client request ID: cd0488d8-da8e-467d-8248-7320c5456b91","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:160]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2462d4d4a5a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n      \"name\": \"466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T07:57:35.126724285Z\",\n      \"endTime\": \"2026-10-12T08:26:42.188209535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n            correlation ID: df9f6d3b-b689-4825-a253-151f9584e7e4\n            client request ID: 5fd0c7fe-22bf-4239-aeb7-b1d20c9c430c\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:160]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2462d4d4a5a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n      \"name\": \"466d7615-8a3d-4982-a2e2-4e4a80210d6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T07:57:35.126724285Z\",\n      \"endTime\": \"2026-10-12T08:26:42.188209535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-5g5xl-j5pmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5hsxwfczjwp5--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-5g5xl-j5pmt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/466d7615-8a3d-4982-a2e2-4e4a80210d6e\n            correlation ID: df9f6d3b-b689-4825-a253-151f9584e7e4\n            client request ID: 5fd0c7fe-22bf-4239-aeb7-b1d20c9c430c","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc252182d240\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c847fqslv6ss\" to finish creating: GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\",\n      \"name\": \"de9482bc-af9d-45d5-a419-3811860ecab1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T03:33:37.063419631Z\",\n      \"endTime\": \"2026-10-12T04:02:41.894551349Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n            correlation ID: 5048e7cb-ed4b-40b9-bfb8-b714b05f3bec\n            client request ID: a436b4d6-2bc5-40f5-a8b1-84635292bdaa\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc252182d240\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c847fqslv6ss\" to finish creating: GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\",\n      \"name\": \"de9482bc-af9d-45d5-a419-3811860ecab1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T03:33:37.063419631Z\",\n      \"endTime\": \"2026-10-12T04:02:41.894551349Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-c847fqslv6ss--managed/providers/Microsoft.Compute/virtualMachines/a2d1d2v0a9x4h6c-np-delete-main-hht52-lmtb9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1330688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/de9482bc-af9d-45d5-a419-3811860ecab1\n            correlation ID: 5048e7cb-ed4b-40b9-bfb8-b714b05f3bec\n            client request ID: a436b4d6-2bc5-40f5-a8b1-84635292bdaa"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109657273612636160","occurred_at":"2026-10-12T14:47:36Z","signature_id":"8a2cb86eeaaac2be5c2b729648eafebf2a78774b306792068dd64d9c83e1f663","pr_number":7311,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","signature_id":"de4fcc56ca6c7e3d2de039fd00e411d285c0c7d0026255cd09fc37bbc4e1d4c7","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109474078921330688","occurred_at":"2026-10-12T02:39:47Z","signature_id":"7e05f12cb3af3d2df3800593d5e4d70e1bcc0ddd1495a395607587d40f14bb85","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109252833235701760","occurred_at":"2026-10-11T12:00:15Z","signature_id":"f33e39eed2f7fbe6ff53acf892c123b972cdb4f2a8fa3a7d7e9f3a49cf743347","pr_number":7219,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108710710912487424","occurred_at":"2026-10-10T00:06:26Z","signature_id":"86e4f5f8163a032f6e683ca40d132f158ebf85db8e38a520bc43b0ee11650f59","pr_number":7165,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377922371850240","occurred_at":"2026-10-09T02:04:03Z","signature_id":"1e52c4ea4ce6b7b96618e329847bb0c9ab97f0ce6ec5f2a8a3add173085fac00","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","occurred_at":"2026-10-08T14:06:48Z","signature_id":"537d2f8688d12100b662592e26040e8091b73d73dada1ea5844df99344a5080b","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"324a1ddefe06745af54ceec3a7cd753b13b17acc2674cc6993eade1b19c7defe","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"7701741dd3b01d9acfabc93cc96d6cf9208adff0af9d2c49651fc30cf2762efd","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"4f28c03f3ffd6f1feafc08f8ba7f2591ca3edbaf7ccebb94d84d8a4050b8d5fe","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"99d703714a8db74b79f0a7f4a791a2c67f0504f0ca749d98ad3677bf89e16f69","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"2f81340bb3a651933dc821aa2ca86669271cdc50691e70e09c6e8aa62b4e76ff","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"8da4047bc4d33387f4b311f92cb1e8ba3a4d59fa7021a9ff0423a80376528f27","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"07ef48ddd58323e88a3c6d921a967ff2a51796e4b4cdf8b03105202a4a8fcc97","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"00d0b2d32441e274eba0d75f1dfcc7751b4d1e6cdc39d82b613431282652086c","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"59ad17797e4c5c9a4a361e9396ecaa021b1de278966eb1d51526ddfdeae7e5fc","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"9c9a5d0f6ef88f93fb5706cfbc62086c704621ae3953320715626c1893a4dfb8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"54b9ecb4dfa7c1ec53a4c0d5518fdcc384dc08d32eb42945364fb76f1fdcc511","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"37771380d1cfd06af04bc60a3e3b334305eddaf4e748f94656e95d0becb4a8ed","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"9e15c77d960faeec1308565e014df7f8752b6fadfa234829868e7060696eefa5","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"a0d80e080c16f8391f653b4f34a80f5db542694bcda517ded36abdc1bf6f5abf","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"a2f41ec8c555bd7bb9e44081752bfbb6e97ad5970facf3a5de491297baa93884","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":21,"runs_affected":21,"failed_runs":21,"run_impact_percent":6.752411575562701,"anchor_occurrences":21,"after_last_push_count":12,"trend_counts":[6,5,7,4,0,0,5,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":32,"prior_last_seen_at":"2026-10-09T20:50:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":21}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-xqmd9dg8sc2s/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-xqmd9dg8sc2s/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4cmzlj277gjf/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4cmzlj277gjf/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-mb49f2vf2mck/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-mb49f2vf2mck/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6ab7f7a7d7318820073a94e892ad474901c481d146387431db8c950325a1baa6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"9578272302c49c5a63e06e79a95c961c925c637a881235d46f047f8f61f08363","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"895d6dc730935dbb7c49cd2e90968b291d7469692edac24ed546ec1e8097382f","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109603048882966528","occurred_at":"2026-10-12T11:12:15Z","signature_id":"bbe6ff38e69c56b0da8c72190329d55c3f58aff3ba835f793e6ee32876e960e6","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109599905499582464","occurred_at":"2026-10-12T10:59:45Z","signature_id":"01ff5d20cf309ae0ced3e300e16a31a96dd5d737827ad14c8718b21686301352","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108661352737280000","occurred_at":"2026-10-09T20:50:18Z","signature_id":"d6d78b4f553ecf032ff0039a772072e8dd415e92575979480f3e3627be38fdcf","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108611234440941568","occurred_at":"2026-10-09T17:31:09Z","signature_id":"7b2af89a3191d6534ece213e75e531df0a43a9214041bfedcb654f1eda579522","pr_number":7444,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"0780aa479fe1109f7c3c28191facf899b2cfa00862be1a670b44484d844ab5f2","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","signature_id":"71021d5ea1fe0872367c0e34afd743c532ccdcbdc2e34685178edf635e296b0d","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317119643914240","occurred_at":"2026-10-08T22:02:26Z","signature_id":"eaa2cead8693174d0c57211952caffdb2c67d10b7dc850a37b0ecc67cb24793b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108291662634553344","occurred_at":"2026-10-08T20:21:17Z","signature_id":"108516cb0647ba3d2f4c87e8257ac1527a6e7121e308f5f5579382713182c90a","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256210653286400","occurred_at":"2026-10-08T18:00:24Z","signature_id":"75c025839b4b77bebd3fbfe7d5be33a670bec55dc16b0fc470c5b7fcd3c0ad20","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","occurred_at":"2026-10-08T14:46:24Z","signature_id":"7a3a06bb18ed74825abe049cd09dc707ee7a940daddfb1a541b6df012fba70c9","pr_number":7105,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","occurred_at":"2026-10-08T08:16:51Z","signature_id":"ffdb06d45831bbbf8246e86634c1c7fa86a1828ef1864f846896e0d1bc90fa06","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","occurred_at":"2026-10-08T07:57:36Z","signature_id":"b122c2e9da305b502bb36b2c546534409e8af338dfddda201b8e60b513ad641d","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"b132c43fd39d685c72ce0a5f22659a72c1f81a5d0a9dd5f857638ee06b041704","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7415/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107928469479362560","occurred_at":"2026-10-07T20:18:05Z","signature_id":"03a6301221c7a6825f5fed75697ef820c87281a3d0a2187bc94a8af35a2207fc","pr_number":7415,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"beeeb8557a906e3bf74511fcdf63017c044aed615160f6a1189ef1547010f8e9","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107836958540566528","occurred_at":"2026-10-07T14:14:27Z","signature_id":"8f4a370df9d0045874c7c3eada9b5a2a1b6adf39b052fcfcd9941efab5980562","pr_number":7227,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107769248779603968","occurred_at":"2026-10-07T09:45:06Z","signature_id":"cdc7d229df7856f7d9198b7ba1b064f7154c9b9b742a95ef9a1ec56ca43c04cb","pr_number":7398,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"cad96447c550c7c633d7082b5c5d1b62094008a4833722c26bba79211fa96628","pr_number":7310,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":5.787781350482315,"anchor_occurrences":18,"after_last_push_count":12,"trend_counts":[6,3,6,2,1,1,5,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":31,"prior_last_seen_at":"2026-10-11T21:12:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":18}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T23:02:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j6774400-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.46:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6774400/providers/microsoft.monitor/accounts/services-j6774400\", namespace=\"klusterlet-2tdgk5uq966q724hns0udujljsi1q0qu\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-nxprp\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tdgk5uq966q724hns0udujljsi1q0qu on cluster ci01-j6774400-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T22:24:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3618048-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.226:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/services-j3618048\", namespace=\"klusterlet-2tdg2drvs0bj4f4vu4h2o5drume4j1rc\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x9h2r\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2tdg2drvs0bj4f4vu4h2o5drume4j1rc on cluster ci01-j3618048-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:44:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1015808-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.213:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1015808/providers/microsoft.monitor/accounts/services-j1015808\", namespace=\"klusterlet-2td5olqiam9j0qubto977v59hbae6bo7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-m2h4p\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2td5olqiam9j0qubto977v59hbae6bo7 on cluster ci01-j1015808-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754180636774400","occurred_at":"2026-10-12T21:12:48Z","signature_id":"54a7b1283931df6f61be82ca63d92bf372c766203c87971d1d688d38deb1c668","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"a67e3b1ce6606aadaaf58795784360bda0ae262c0ccd8be147cd4aa4d8cfa336","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109566058561015808","occurred_at":"2026-10-12T08:45:16Z","signature_id":"16da3c90424d3e20001b8e7c2d884bfc967a23a24761b0cf7cc40ac186996cdd","pr_number":7219,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528983438626816","occurred_at":"2026-10-12T06:17:57Z","signature_id":"9a0460e320537545b7df7ed4ea07d8af2e7eacb2bec27a23e38614a9b14396d9","pr_number":7458,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","signature_id":"7f1c703708a0c73e567c797b757f2be0ee02cc20a38341bda264d7aaee221d0e","pr_number":7424,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109391747573354496","occurred_at":"2026-10-11T21:12:37Z","signature_id":"128d05881de79fd499086d5693a2febe7c68869a9b1927ffc1becfa9da92767e","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108886756110635008","occurred_at":"2026-10-10T11:45:56Z","signature_id":"569ddb71f13f546121c7971b77e866f8e636b91dfed46d5b29bdb27c60834dd7","pr_number":7374,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694589719187456","occurred_at":"2026-10-09T23:02:22Z","signature_id":"ce44dc83babfef2811856faa5dcaa4ffd6178cd17bba577f8a5125d2ff96ccb0","pr_number":7227,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","occurred_at":"2026-10-09T21:37:50Z","signature_id":"7d1a3870cd1a33572d8e26c0f1e350eca9e57ea685800f4e196987e73f6c5d34","pr_number":7446,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256282895978496","occurred_at":"2026-10-08T18:00:39Z","signature_id":"5a7e6f416bdd814aa6842042551270c54b63893bd3ddd65b8fe96afa706bcb83","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"9a5683b6fb4e970df594707347166c3a5464fde6e4de17ed87d6680996438cdc","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"13fd8fddd2d9f41c17b15476dbb1a6d18d3c125423d8872d5b4d0619fa486992","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","occurred_at":"2026-10-08T11:09:46Z","signature_id":"e82e51d2e215239d7b4cd935e809cb5992c44a5d389b3abdcf1a7fc37def35f8","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108101483491758080","occurred_at":"2026-10-08T07:45:34Z","signature_id":"afb48a8584f79bbdd2eee32217731be00e00c63d7874b7427fe12a549c0c7d4a","pr_number":7378,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","signature_id":"17502e6a782e32cf8d05639323afa25f0eb69d1c67087752a4cd2f50e385cc87","pr_number":7412,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107979674851217408","occurred_at":"2026-10-07T23:41:33Z","signature_id":"55900fd3bc28e32d9a639ce8648efcc35e3932eddeb80af6e6b043605ce1b905","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107955248256520192","occurred_at":"2026-10-07T22:04:29Z","signature_id":"b6691d6bcc2d313c3a62c74fbf150224b295d130f055782982141b1f34547850","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916465603612672","occurred_at":"2026-10-07T19:30:23Z","signature_id":"8869545ed690b35be454db0cd6dff2e92fefd57b72a3ff6445d159a8e93031e9","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":18,"runs_affected":14,"failed_runs":14,"run_impact_percent":4.501607717041801,"anchor_occurrences":18,"after_last_push_count":20,"also_in":["PROD","STG"],"trend_counts":[39,16,0,2,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":87,"prior_last_seen_at":"2026-10-09T02:04:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullc5dl28vw558j.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34f9246db800\u003e: \n    VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x34f9246db6e0\u003e{\n            msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x34f9246db5e0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullc5dl28vw558j.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34f9246db800\u003e: \n    VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x34f9246db6e0\u003e{\n            msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x34f9246db5e0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullctpjh66q2dr2.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0x97759ef0500\u003e: \n    VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x97759ef04a0\u003e{\n            msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x97759f21ec0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullctpjh66q2dr2.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0x97759ef0500\u003e: \n    VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x97759ef04a0\u003e{\n            msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x97759f21ec0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2de8dabda108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-v5jj9lf2wr67\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"name\": \"c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:20:36.247617787Z\",\n      \"endTime\": \"2026-10-07T18:49:41.673001167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2de8dabda108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-v5jj9lf2wr67\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"name\": \"c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:20:36.247617787Z\",\n      \"endTime\": \"2026-10-07T18:49:41.673001167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377922371850240","occurred_at":"2026-10-09T02:04:03Z","signature_id":"c1431ac14fd0271bd4bc507228550d48034fc8fe91d5f2acb2d3e2d60688b09f","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"f23553b8fe3543a67ee1e3d8aafdef51bf55d49055f856784748065ac9884238","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"6606f0c53685d57eac73a261b99ec8523294d02e9f04d706f5486f1b88272365","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"e6140a91cf58e1093a15df9527c4955caccd5e340b860748578f39c12af14e5e","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"5ceb481507f4ca52c699d4ee4b5bc627642caa14a2e4b131a7ee8c113d7edf2c","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"16e437d19740573dbce777293dfb290c1261ca41fa7793782e50c0650b0ff035","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"83affcfb11dd475414f0e926b53f754c59b86fee5dfc1b34515c53c67abd39b8","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"5c68c137d6e10f757f06aa245dbad8a30058f94e21fab5889b2c912a02342357","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"57b1884bac955ab5f382942dfb3a03cdad30c1cd1fa31f46943b54e6a0ebe071","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"c0d8513a01d197d2a0670900409b40efbe2c6c80538e5b9afafd3a6a0d810067","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"9d1a449c54137967e0ef922ae6fb2f6b6819636ed301e6416ddd77db7dab6415","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"42242f0736ff6b2cabbb937ba4f48e4f3ed306ea01fcfdba4459ac3cfb832b7f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"7b412a98d9ede076c1a2df7a7038043eb6df6fa1023a0a7a3882d3027c3aaf83","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"ea119913577e1401aead2826975e251b0b4dfe27af9566d73b22dca408a22619","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"4b1fb5b920b17720b82999c711081f2a10ec1c9d9ea8ed05033b5998a486bb39","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"ee6bea4d8f68ba09c2d18707029359aa47b377c9594af345abe1ae957945edfd","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"83b0bcfc35f4e8dd7f58f5584c061cc026360328425f7ff053e8d20e993e57f4","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"e71983d5997d779d858dc33d7be7f632aa394ef38c01af01f25da4318397c4ab","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":16,"runs_affected":9,"failed_runs":9,"run_impact_percent":2.8938906752411575,"anchor_occurrences":16,"after_last_push_count":90,"trend_counts":[0,3,4,6,0,0,3,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":160,"prior_last_seen_at":"2026-10-09T11:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3aa92878a060\u003e: \n    resource group \"hcp-metrics-w7n8h467qbnb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-w7n8h467qbnb\" to finish deleting: GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\",\n      \"name\": \"eb261c81-ca83-45b2-bdfd-0de40227c103\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:58:20.608938335Z\",\n      \"endTime\": \"2026-10-12T16:22:21.07258074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5jcbgo4cpr0su6el4fc6i0lag0dj still exists (deletion dispatched at 2026-10-12T15:58:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n            correlation ID: 9da4d8e2-ebfd-4094-bf43-c7701d90768a\n            client request ID: 61ad067a-2c1b-43a6-b798-69bd268d4471\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3aa92878a060\u003e: \n    resource group \"hcp-metrics-w7n8h467qbnb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-w7n8h467qbnb\" to finish deleting: GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\",\n      \"name\": \"eb261c81-ca83-45b2-bdfd-0de40227c103\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:58:20.608938335Z\",\n      \"endTime\": \"2026-10-12T16:22:21.07258074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5jcbgo4cpr0su6el4fc6i0lag0dj still exists (deletion dispatched at 2026-10-12T15:58:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4566144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb261c81-ca83-45b2-bdfd-0de40227c103\n            correlation ID: 9da4d8e2-ebfd-4094-bf43-c7701d90768a\n            client request ID: 61ad067a-2c1b-43a6-b798-69bd268d4471\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x1476512fa078\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n      \"name\": \"d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:58:47.33491695Z\",\n      \"endTime\": \"2026-10-12T16:22:51.509357112Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb58jqqdu8adgelm46n5haqq88idqc still exists (deletion dispatched at 2026-10-12T15:58:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n            correlation ID: f7d4d296-e895-4942-ace3-b59e68657889\n            client request ID: 314b1f02-1204-4002-a941-604f43078239\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x1476512fa078\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xjg677xdgtps\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n      \"name\": \"d6de2ca4-02ff-4736-bb7c-5d1de1b40369\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:58:47.33491695Z\",\n      \"endTime\": \"2026-10-12T16:22:51.509357112Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb58jqqdu8adgelm46n5haqq88idqc still exists (deletion dispatched at 2026-10-12T15:58:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6de2ca4-02ff-4736-bb7c-5d1de1b40369\n            correlation ID: f7d4d296-e895-4942-ace3-b59e68657889\n            client request ID: 314b1f02-1204-4002-a941-604f43078239\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x351b8e4ae138\u003e: \n    resource group \"disabled-image-registry-pzs7vs68g9xt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-pzs7vs68g9xt\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\",\n      \"name\": \"f1e860d3-9723-4468-b91f-a96474515d13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:57:28.987630883Z\",\n      \"endTime\": \"2026-10-12T16:21:31.7469599Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5a4g859cjm0hvqmru36k5u916p18 still exists (deletion dispatched at 2026-10-12T15:57:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n            correlation ID: eab286f4-49cc-48ea-8ec4-6c70b43ff1d0\n            client request ID: 1a74a385-2e63-4deb-a488-b16ad26cc354\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x351b8e4ae138\u003e: \n    resource group \"disabled-image-registry-pzs7vs68g9xt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-pzs7vs68g9xt\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\",\n      \"name\": \"f1e860d3-9723-4468-b91f-a96474515d13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:57:28.987630883Z\",\n      \"endTime\": \"2026-10-12T16:21:31.7469599Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdb5a4g859cjm0hvqmru36k5u916p18 still exists (deletion dispatched at 2026-10-12T15:57:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e860d3-9723-4468-b91f-a96474515d13\n            correlation ID: eab286f4-49cc-48ea-8ec4-6c70b43ff1d0\n            client request ID: 1a74a385-2e63-4deb-a488-b16ad26cc354\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109659008674566144","occurred_at":"2026-10-12T14:54:37Z","signature_id":"aa1b645215452d614b0ab2fb286d0f4c2a52df3c3bf4b0684396494f99dc7271","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"9464cf8289c40c4ace5feeaa2af2e60f2ff16f8a6b1306bce054078519a3b607","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"c8fe87bb1fbea1d2b82daec5c2abcf5f379166a6c5ecbc60a09403cacb34657b","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"558c8187820f4c407e61afc39a35aa683527faaae3460d474421fd2b801bb83c","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"bd0bca41ebd669a1d4d1dbf0424b19b876b1622f03854960d3114e98009f90b0","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"0772c78b685bdb878b9ff0b3ec87a5c4bc3da82c58ac34bdef13e588a7eb7ab1","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"cbd3d2251edc52984c4b4261a1e14d0463ccc02836f82b3f5018bb0bc74c6def","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"dd3b9534d3561884612cb8845e38cad44c65e45133a439f30559f4a27a034617","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","occurred_at":"2026-10-09T02:04:14Z","signature_id":"ca52d27568368e0703bdb7fcdb2e8b54933f66f559908b6d0a48d5691bc53d6b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108311064448864256","occurred_at":"2026-10-08T21:38:22Z","signature_id":"87480de3ded9bd25bdf3480fe553bfa947aa5a80228ef448c08c54751c457c7d","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","signature_id":"0e4b25ccea79ae9ce774b1501e6333b6a44367fc8f0e789e5ce189b4d807484c","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","signature_id":"7cbece22670a9f172523e3b354735cd562db62f7e42f6e03ee9317ebdd207c57","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","signature_id":"d6f3dbb1bf15144cc0a62926bd3e48a5cae1b7b619360a81ed11fa37f85d80e3","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"004e0ec3888a99aaedcb7c2531a56de72dc32aa9559eb821f4fe7792ca5441d3","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"535c72aaa4c0665d8f64e9446dca3aa8e6d1fb91f52bd8fc1875bce44908d6ed","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"85866f3cd23f94e7a49f5e7cddfef735342e7f7a3e10ff85febdbe3d9afe74fe","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":12,"runs_affected":12,"failed_runs":12,"run_impact_percent":3.8585209003215435,"anchor_occurrences":12,"after_last_push_count":11,"trend_counts":[5,4,8,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":34,"prior_last_seen_at":"2026-10-08T22:02:45Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":12}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T00:16:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0504704-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0504704/providers/microsoft.monitor/accounts/services-j0504704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5fdbbd5c5b-l6bt7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j0504704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T00:16:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0504704-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0504704/providers/microsoft.monitor/accounts/services-j0504704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5fdbbd5c5b-l6bt7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j0504704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T00:17:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4990592-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.37:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4990592/providers/microsoft.monitor/accounts/services-j4990592\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c59bdf7c8-spr9r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j4990592-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T00:17:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4990592-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.37:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4990592/providers/microsoft.monitor/accounts/services-j4990592\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c59bdf7c8-spr9r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j4990592-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T20:22:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8343424-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.84:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8343424/providers/microsoft.monitor/accounts/services-j8343424\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68b89b8957-rg9qs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j8343424-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T20:23:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8343424-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.84:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8343424/providers/microsoft.monitor/accounts/services-j8343424\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68b89b8957-rg9qs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j8343424-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","occurred_at":"2026-10-08T22:02:45Z","signature_id":"478c229f000373303094e502fe5867ab11bb46818c55fe4914e070c3ce22b112","pr_number":7389,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","occurred_at":"2026-10-08T22:02:25Z","signature_id":"23b4c1fcca9c095c3310a93e598a980361364ffd7335fed71ecdd85117fddf7e","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","occurred_at":"2026-10-08T18:00:34Z","signature_id":"4f9f8db4cb3f25b70a751ee41026225f752fc0d2a21726d68e44cf77af9d8f9f","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108248385097043968","occurred_at":"2026-10-08T17:29:18Z","signature_id":"293fb0dcb3ed0945c0a68b097fa102ed57356450154c2fa182d4761ee486de09","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108221120938250240","occurred_at":"2026-10-08T15:40:58Z","signature_id":"27e681f75202e09c64bf1bc1b442c563ec7f3137ae918747ec39eefd467cec74","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","occurred_at":"2026-10-08T13:01:55Z","signature_id":"71513c99645f5a7b46d22a8d1780149946d3f4d432f0cff96d7e41203c90b083","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","signature_id":"b3859c09791b6a1dee073ca99210a9b01905c300d49723024d404dda969c5d31","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","occurred_at":"2026-10-08T09:41:34Z","signature_id":"0505f57a41fb3f2b5cc475246473e4503d1def678146b025f446120e420b23ae","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107979694195347456","occurred_at":"2026-10-07T23:41:38Z","signature_id":"a9f42fb8cd948b493ad952c111be2d077f8870c40915de4addfd8f530e496344","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"7263867785ff09df80dda0390383f3d5a769b78cdf799960e2fb0de6363a6a4c","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"63861b0edd2874dd78327b3b31697a4dfd50630052ae8d5d23132d2a7630b468","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1a734d7fe3d2c29c8e29a73b6bae5924a4a747ab2baa59c0a15b4bb9931650a5","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.9646302250803859,"anchor_occurrences":12,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,12,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"External auth issuer validation rejects an invalid external auth issuer URL without leaving a child or changing the shared cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Immutable cluster configuration rejects an immutable cluster DNS prefix PATCH without changing the shared cluster","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Immutable node pool configuration rejects an immutable node pool OS disk size PATCH without changing the shared node pool","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Node pool create validation rejects a negative replica count without creating a node pool or changing the shared cluster","occurrences":1}],"full_error_samples":["provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------","provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------","provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"MultipleErrorsOccurred\",\n    \"message\": \"2 errors occurred\",\n    \"details\": [\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Too long: may not be more than 15 bytes\",\n        \"target\": \"id\"\n      },\n      {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n        \"target\": \"id\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":11,"runs_affected":8,"failed_runs":8,"run_impact_percent":2.572347266881029,"anchor_occurrences":11,"after_last_push_count":2,"trend_counts":[0,1,1,8,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-09T11:02:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x643438c6138\u003e: \n    resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n      \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n            correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n            client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x643438c6138\u003e: \n    resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n      \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n            correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n            client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x87a272e80c0\u003e: \n    resource group \"autoscaling-cluster-dch49t4bvhmd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dch49t4bvhmd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n      \"name\": \"314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:11:02.340187908Z\",\n      \"endTime\": \"2026-10-09T12:35:03.752595501Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x87a272e80c0\u003e: \n    resource group \"autoscaling-cluster-dch49t4bvhmd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dch49t4bvhmd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n      \"name\": \"314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:11:02.340187908Z\",\n      \"endTime\": \"2026-10-09T12:35:03.752595501Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3f620761c090\u003e: \n    resource group \"customer-rg-lbqh9vzm8skr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lbqh9vzm8skr\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\",\n      \"name\": \"ff9dce59-52f3-4407-ad07-f8872d204253\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:06:09.323431375Z\",\n      \"endTime\": \"2026-10-09T12:30:13.733939626Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3f620761c090\u003e: \n    resource group \"customer-rg-lbqh9vzm8skr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lbqh9vzm8skr\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\",\n      \"name\": \"ff9dce59-52f3-4407-ad07-f8872d204253\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:06:09.323431375Z\",\n      \"endTime\": \"2026-10-09T12:30:13.733939626Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"4ea81d2e607f663f33592f25d4328ddf786e7ed343f4f03175a81981d5aeb666","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","signature_id":"cbfda8e35f43a33bada09d676dd317e056f0425610ce818b8ca139686e99b4bb","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","signature_id":"fd5af6bc3f19a3de8c7e9853e809e76590e6bfa6137c67e04556267aa2c73d5b","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"02a2305632d97b7f1a474a433f8142ff13828b0ad53edb43a5c977da2a5c4bb6","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"2e5e4c7c8eb64f44eb3b1af95f10974b102ff0871033df3fb3fbe2fb1f9323bb","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"746988fc7364fd70b66c30cfad0f0919e60becbcdc601bb071a2a47a33663ad3","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"8d0e0342cbc9e23b844c7aca35ed99507754ea217dedb85ce43d4f14fe551007","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","occurred_at":"2026-10-09T02:04:14Z","signature_id":"c8ae875f9675b060f7ff6177afd27d7e1577f85c03248d7c33bede62663dad07","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"f5a624ce9e80c1ff18f16e6ce00681fccca997dd1474f86b90ce8aa6c5f750b3","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","occurred_at":"2026-10-08T14:46:24Z","signature_id":"13da5141ccb9f5269ca39d7846b3675b49c796a19d5d81feddf02f4d488d863f","pr_number":7105,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"df2ee66aadf49a7254b9c095a9ada5b4b3e24662144e3c9aaa040cca41949609","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":10,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":10,"after_last_push_count":4,"trend_counts":[0,0,4,2,0,0,4,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-09T09:35:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":5},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert collection is complete","occurrences":5}],"full_error_samples":["alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","occurred_at":"2026-10-09T09:35:37Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7226,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","occurred_at":"2026-10-09T09:35:37Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7226,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7226,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7226,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":2.8938906752411575,"anchor_occurrences":9,"after_last_push_count":11,"trend_counts":[5,1,2,2,0,0,4,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":43,"prior_last_seen_at":"2026-10-09T22:55:44Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":9}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T21:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"cloud-network-config-controller-54f58dd464-7xc6g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d482c703-1325-4911-a179-7fd115deb112\"\n  Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller-54f58dd464-7xc6g (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T21:56:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"cloud-network-config-controller-54f58dd464-7xc6g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d482c703-1325-4911-a179-7fd115deb112\"\n  Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller-54f58dd464-7xc6g (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T21:57:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"csi-snapshot-controller-746f546586-q8jss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8db2a082-8106-487f-b148-74ed6ac93307\"\n  Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/csi-snapshot-controller-746f546586-q8jss (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T21:59:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"csi-snapshot-controller-746f546586-q8jss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8db2a082-8106-487f-b148-74ed6ac93307\"\n  Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/csi-snapshot-controller-746f546586-q8jss (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T19:45:53Z\n  Ended:       2026-10-12T20:13:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9ddee187-d906-4221-b48f-73e1102c7b1e\"\n  Description: Pod ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T19:45:53Z\n  Ended:       2026-10-12T20:13:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9ddee187-d906-4221-b48f-73e1102c7b1e\"\n  Description: Pod ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T13:08:11Z\n  Ended:       2026-10-12T13:34:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d015a5d-c4df-4754-bdd2-c1b853805f1f\"\n  Description: Pod ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T13:09:11Z\n  Ended:       2026-10-12T13:33:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d015a5d-c4df-4754-bdd2-c1b853805f1f\"\n  Description: Pod ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"d4b2e29eb71d7ac5c954c156d7e1d4a7c4f83854fd89b2f4bf826b82158948d1","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"a25524a009142bcd67f70022a288b9e9be2947605657927be7457aeb02bcc1d2","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"ccfda79db149eab1089427f6b12af29fcb8d7b25163196bd92ebcab836f644a6","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"e9c8652d1165ca174ad5201f1063a5d60e6d9e1e776a81a9de4becc5e6a83ae5","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7448/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108692920105177088","occurred_at":"2026-10-09T22:55:44Z","signature_id":"28fcb1c27858e1387aacc2dacf213e3c55e8e38d44b6bf8ef602a149e90b5993","pr_number":7448,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","occurred_at":"2026-10-09T06:02:56Z","signature_id":"13066145b3c69b6571c75ae201d660362eb9c50399b3ea171948ebe6c00d48cf","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"2d0718dd6bc944ecb1f8e90e435ead1ad5ebb1657342b129e97e84fc23cf0c63","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"e76d176cb58c095e589bc218c3e104c756d14a6ceb09fa88d761f62736c4f607","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"93daf971eb2e3d807bd5be43c5947b64fbc568467906d318a73562ccf23adc89","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":9,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.9292604501607717,"anchor_occurrences":9,"after_last_push_count":4,"trend_counts":[0,3,1,5,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":18,"prior_last_seen_at":"2026-10-09T11:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x715418a8198\u003e: \n    resource group \"rg-hs-pre--d8fntzv77rg5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6jnw6j\" in resourcegroup=\"rg-hs-pre--d8fntzv77rg5\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\",\n      \"name\": \"1f6364b4-168b-4408-93de-ce5f903b20aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:05:02.733087351Z\",\n      \"endTime\": \"2026-10-09T12:29:06.620380667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kg3h59n4o9tks40jdsh5l5jro8g still exists (deletion dispatched at 2026-10-09T12:05:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x715418a8198\u003e: \n    resource group \"rg-hs-pre--d8fntzv77rg5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6jnw6j\" in resourcegroup=\"rg-hs-pre--d8fntzv77rg5\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\",\n      \"name\": \"1f6364b4-168b-4408-93de-ce5f903b20aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:05:02.733087351Z\",\n      \"endTime\": \"2026-10-09T12:29:06.620380667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kg3h59n4o9tks40jdsh5l5jro8g still exists (deletion dispatched at 2026-10-09T12:05:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x52f9b8920c0\u003e: \n    resource group \"cni-cilium-lqwdfr6cjvht\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-lqwdfr6cjvht\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n      \"name\": \"d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:57:36.843017136Z\",\n      \"endTime\": \"2026-10-09T12:21:46.074977182Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb86qasjq198a6vq4pknlkm4b3grgda still exists (deletion dispatched at 2026-10-09T11:57:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x52f9b8920c0\u003e: \n    resource group \"cni-cilium-lqwdfr6cjvht\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-lqwdfr6cjvht\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n      \"name\": \"d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:57:36.843017136Z\",\n      \"endTime\": \"2026-10-09T12:21:46.074977182Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb86qasjq198a6vq4pknlkm4b3grgda still exists (deletion dispatched at 2026-10-09T11:57:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x4a16d2b40f0\u003e: \n    resource group \"rg-candidate-5-1-trdpp8hx8tpd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jjf49v\" in resourcegroup=\"rg-candidate-5-1-trdpp8hx8tpd\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n      \"name\": \"292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T03:05:28.202107106Z\",\n      \"endTime\": \"2026-10-09T03:29:29.34914283Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dh1i1h74jbtnnmtcppkg9ambqd8h still exists (deletion dispatched at 2026-10-09T03:05:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x4a16d2b40f0\u003e: \n    resource group \"rg-candidate-5-1-trdpp8hx8tpd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jjf49v\" in resourcegroup=\"rg-candidate-5-1-trdpp8hx8tpd\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n      \"name\": \"292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T03:05:28.202107106Z\",\n      \"endTime\": \"2026-10-09T03:29:29.34914283Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dh1i1h74jbtnnmtcppkg9ambqd8h still exists (deletion dispatched at 2026-10-09T03:05:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"da09c37462afa5025b563af86b7fe1d3b933772f65cffb7c0600dea0aa30b85d","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"85e24c7518789cbd7818e3dc2cced879b06d0785625f57d40429c77d0b688b8b","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"617a9de46bb1c7108902990d2a726a9e7ea6051560c6bfa5ada743c1192b3686","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"8249b8d9f8856d073b9d1060e96a79d6fd51a11374782dee26cbd35f7ce25db2","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"ed915b16f8b2b776750f739a5daa97de703f32658ba93ea9faf3a154c2ab7b11","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108214686754279424","occurred_at":"2026-10-08T15:15:21Z","signature_id":"80c9f90b48cd6de845d1ea9fcfc0ec8a6a97df54de2e0676d98bab3ffdbc932e","pr_number":7305,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"775e9e34036b536418ecb6ce7f646ee64060e4bac63d0b2703f2cb5c873f567c","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"a3363c6da7bde363b2cf3af699a41f3e3396a095df565f4aedc0f37273fedad8","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"9ee241ee8ad2e0f0ccaa82ff16aea7dd50b95471966e2fca705a2a6547db1ebe","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":2.2508038585209005,"anchor_occurrences":7,"after_last_push_count":4,"trend_counts":[2,1,2,1,1,1,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-11T23:29:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb6d00db1440\u003e: \n    failed to create NodePool np-kxb95g: failed waiting for nodepool=\"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\" in resourcegroup=\"rg-hs-pre--x6fpqw6hqtjb\" to finish creating: GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n      \"name\": \"dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:32:07.510834189Z\",\n      \"endTime\": \"2026-10-12T20:01:15.571433072Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n            correlation ID: bfbdef2c-b02a-48b8-a68a-a2439acfc443\n            client request ID: 7e134c20-557c-48a5-9100-cbcf934467d5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb6d00db1440\u003e: \n    failed to create NodePool np-kxb95g: failed waiting for nodepool=\"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\" in resourcegroup=\"rg-hs-pre--x6fpqw6hqtjb\" to finish creating: GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n      \"name\": \"dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:32:07.510834189Z\",\n      \"endTime\": \"2026-10-12T20:01:15.571433072Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n            correlation ID: bfbdef2c-b02a-48b8-a68a-a2439acfc443\n            client request ID: 7e134c20-557c-48a5-9100-cbcf934467d5\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n      \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n            correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n            client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b569c9d54e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlcxj9z6srvl\" to finish creating: GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"name\": \"5a974196-cc3e-4c19-83bb-20557d537e04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T00:41:24.479383887Z\",\n      \"endTime\": \"2026-10-12T01:10:29.269134441Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6174848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a974196-cc3e-4c19-83bb-20557d537e04\n            correlation ID: 2bffc39b-cd83-4954-a0a9-c7daa45e4979\n            client request ID: 8e77b332-cff0-4849-90bf-1e7ad363de44\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5b5e959eb80\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hxz5lhgx6pgc\" to finish creating: GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"name\": \"bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T09:08:14.887674935Z\",\n      \"endTime\": \"2026-10-10T09:37:18.731959559Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n            correlation ID: 85344fe9-db3b-408b-ba74-a3aed4f6938b\n            client request ID: 900c9a16-ab81-46bf-aba5-3f67f267761a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5b5e959eb80\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hxz5lhgx6pgc\" to finish creating: GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"name\": \"bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T09:08:14.887674935Z\",\n      \"endTime\": \"2026-10-10T09:37:18.731959559Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8557440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8eab11-2b4f-4ee5-bf79-f2c3f0cf511f\n            correlation ID: 85344fe9-db3b-408b-ba74-a3aed4f6938b\n            client request ID: 900c9a16-ab81-46bf-aba5-3f67f267761a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109708565542342656","occurred_at":"2026-10-12T18:11:33Z","signature_id":"8d1213d4562f39353e445c7a58b0fd1c57fd1080920c955cf4d994b8f4a3de3c","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109426221556174848","occurred_at":"2026-10-11T23:29:37Z","signature_id":"0c85b612becba4c571d22661fd122ecdb27d938385ea00a4fd07f500c5e96552","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108830741738557440","occurred_at":"2026-10-10T08:03:23Z","signature_id":"d2c8e054823fbd376c984214c001bc93b2a900e7a4a75bde428fa6fea3d24178","pr_number":7227,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","signature_id":"d49e6094505b38289976eaa29d7017c02d2680fb4b1969d63ddd5a93cf4da739","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108182743610298368","occurred_at":"2026-10-08T13:08:28Z","signature_id":"2ab2afb758add7ee8595134f90992b6ec46e194592da1dee4e4a8da09cd82ffb","pr_number":7403,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025089978011648","occurred_at":"2026-10-08T02:42:01Z","signature_id":"898639e46d512657722f4281a7d0dfb4428d8ed1240fc047c4055963519f6d5a","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107872947002675200","occurred_at":"2026-10-07T16:37:27Z","signature_id":"6aa7421e150db2feff9236efdb495765f16c29a20bce67118d40673f019a35dc","pr_number":7310,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":2.2508038585209005,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,2,2,2,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-09T11:01:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x72d88398300\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n            correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n            client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x72d88398300\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n            correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n            client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ae2e5f963c0\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2c747kttkrf7\": PUT https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2c747kttkrf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ae2e5f963c0\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2c747kttkrf7\": PUT https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2c747kttkrf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x95c6dd09a20\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zsv74hnjtfn5\": PUT https://rp.j2592384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-zsv74hnjtfn5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x95c6dd09a20\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zsv74hnjtfn5\": PUT https://rp.j2592384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-zsv74hnjtfn5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109643872534532096","occurred_at":"2026-10-12T13:54:09Z","signature_id":"1f4d7a78c6e6f2b64bfcac7b4fc7dadaf9ee3793c7757ef0699936b0dd1911ad","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","signature_id":"3c56fcfecfdfd7699d62c207a6ece2f62b01d37359d14f8eaa7e27e10724c8d1","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","occurred_at":"2026-10-09T07:02:15Z","signature_id":"ea6e53cca656512f02d19790e724606aed7efaeb47b605b3a33d11d36c819c38","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108316538086887424","occurred_at":"2026-10-08T22:00:07Z","signature_id":"7e465cdb4b8db885d2f1d5a2350e3f71d22bf5070b017c7982cfbb0f329e79aa","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108178486299463680","occurred_at":"2026-10-08T12:51:33Z","signature_id":"703ff58f90b61266d47daa64e91c49cac059b2fddfc664d613b54ef26eabfb02","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"b243e7b21db9c6849fd992f8e4574d0ab72105b36c17e4d86b14eba547a70a25","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"80e9dbbbafdd3fbd1f829013bf1ed0880b1a13d485e1a90cecc503401b7f6c2d","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":2.2508038585209005,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,4,3,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-08T16:11:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xed922aab58\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xed9195f1e0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0xed91fd22c0\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xed922aab58\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xed9195f1e0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0xed91fd22c0\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x7b70a422fc0\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x7b70a4e87a0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x7b70a6fbd20\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x7b70a422fc0\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x7b70a4e87a0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x7b70a6fbd20\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1e29e0f51bc0\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1e29e2b846a0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x1e29e176e4f0\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1e29e0f51bc0\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1e29e2b846a0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x1e29e176e4f0\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","signature_id":"a43f99b00f1bbd4a520785c52599fad0e47aab9b8859a5d6eb181cda1291d1d6","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","occurred_at":"2026-10-08T07:02:22Z","signature_id":"3cc6e00f4b908acb0ea62e41044f8a94e6ce5698be960fa38a7f85470bd35cba","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","signature_id":"38cafc0c9d53e48809e4f80070514d02cab460de26c498c2ee86e36255eb9c93","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107970662109810688","occurred_at":"2026-10-07T23:05:44Z","signature_id":"94db297794b5684fabf014b75d93616ed7e1a167578aa35ce2524a04ff1dbadd","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"787ad618025d031d81ee980f6b67a1bd19c9afd249e15e2fd07410bfe1d4ae20","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"50c5be6aacbbb8af14d88b4bc3038a15509228c93ec4dc6e779b9e4c9fd79f5a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1faf7d5cc80b0f7cbde2eb8dbba3d93237d698cd5407434495000a5ca371a54d","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.9292604501607717,"anchor_occurrences":6,"after_last_push_count":3,"also_in":["INT"],"trend_counts":[1,0,2,2,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-09T07:25:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x2f223f7aa0d8\u003e: \n    resource group \"hcp-metrics-lh6cn9z9mpvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-lh6cn9z9mpvb\" to finish deleting: GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n      \"name\": \"666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T13:00:52.055384192Z\",\n      \"endTime\": \"2026-10-12T13:24:57.118445449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td8hkede6pik7afuli4rqulo2mdm0ib still exists (deletion dispatched at 2026-10-12T13:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n            correlation ID: 32f74568-8662-4b4e-8805-1aefdc453b8a\n            client request ID: 470d44cf-4554-4545-aed1-856e45d54cdb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x2f223f7aa0d8\u003e: \n    resource group \"hcp-metrics-lh6cn9z9mpvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-lh6cn9z9mpvb\" to finish deleting: GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n      \"name\": \"666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T13:00:52.055384192Z\",\n      \"endTime\": \"2026-10-12T13:24:57.118445449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td8hkede6pik7afuli4rqulo2mdm0ib still exists (deletion dispatched at 2026-10-12T13:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n            correlation ID: 32f74568-8662-4b4e-8805-1aefdc453b8a\n            client request ID: 470d44cf-4554-4545-aed1-856e45d54cdb\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0xde424e4090\u003e: \n    resource group \"admin-credential-lifecycle-test-7p8rt54bzxxl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-cm6t76\" in resourcegroup=\"admin-credential-lifecycle-test-7p8rt54bzxxl\" to finish deleting: GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\",\n      \"name\": \"d217aaa4-6911-41fe-a395-709b350055ad\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:20:06.890936008Z\",\n      \"endTime\": \"2026-10-12T12:44:10.847761888Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td7rmojjop6e5ni0mk3h0r2c95v0aqq still exists (deletion dispatched at 2026-10-12T12:20:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n            correlation ID: 5a65ad43-439a-49ac-a5b3-3ea36ef8c6fd\n            client request ID: 3b679075-36d2-4af2-ad13-7fdd7af0d4bb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0xde424e4090\u003e: \n    resource group \"admin-credential-lifecycle-test-7p8rt54bzxxl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-cm6t76\" in resourcegroup=\"admin-credential-lifecycle-test-7p8rt54bzxxl\" to finish deleting: GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\",\n      \"name\": \"d217aaa4-6911-41fe-a395-709b350055ad\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:20:06.890936008Z\",\n      \"endTime\": \"2026-10-12T12:44:10.847761888Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td7rmojjop6e5ni0mk3h0r2c95v0aqq still exists (deletion dispatched at 2026-10-12T12:20:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n            correlation ID: 5a65ad43-439a-49ac-a5b3-3ea36ef8c6fd\n            client request ID: 3b679075-36d2-4af2-ad13-7fdd7af0d4bb\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x126a68f88048\u003e: \n    resource group \"nodepool-update-nodes-ppzw4fscb65q\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ppzw4fscb65q\" to finish deleting: GET https://rp.j7543168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\",\n      \"name\": \"5402a6f6-777b-405f-bf42-17f6202e880d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T09:00:29.162024664Z\",\n      \"endTime\": \"2026-10-09T09:24:36.108080611Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb5gs6ackm9riurrcuq3gadk7un22q7 still exists (deletion dispatched at 2026-10-09T09:00:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x126a68f88048\u003e: \n    resource group \"nodepool-update-nodes-ppzw4fscb65q\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ppzw4fscb65q\" to finish deleting: GET https://rp.j7543168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\",\n      \"name\": \"5402a6f6-777b-405f-bf42-17f6202e880d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T09:00:29.162024664Z\",\n      \"endTime\": \"2026-10-09T09:24:36.108080611Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb5gs6ackm9riurrcuq3gadk7un22q7 still exists (deletion dispatched at 2026-10-09T09:00:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"f1b1b884321d5b2ac6d5d82d2bb343fddd2a8c66964ab52673d16c25eb8d5697","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"5b41a952bcf6d2d929fc1668bfb207665a25d88b8be5793b80fd4bb0cf28e575","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108458935697543168","occurred_at":"2026-10-09T07:25:58Z","signature_id":"9c17d7d30e387ba5b6997b09f0c0488b684608a6ad4866054d6d83675e245a1b","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108409414854643712","occurred_at":"2026-10-09T04:09:10Z","signature_id":"683f0dfc089fd69b29c305b21fb888467085c261c34597202abd78d953a0cf3b","pr_number":7105,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"b9572568edad6e488a7ae0195c7620f5eca412060505066f9072aea06ab756cc","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","signature_id":"8db99276b1e37ca2b73c952dfac288a0b7ae6c24b3ed28e03f5e617ef6f4ab9c","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.9292604501607717,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[5,2,1,1,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":58,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":6}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:37:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-vk9xs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-vk9xs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:42:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-64b5859f7f-5glfr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod sessiongate/sessiongate-64b5859f7f-5glfr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:42:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"fleet\", pod=\"fleet-controller-76c4b4c8bc-8gvkr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod fleet/fleet-controller-76c4b4c8bc-8gvkr has been in a non-ready state for longer than 5 minutes.","alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:30:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14292482-vmss000002-dskt8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-14292482-vmss000002-dskt8 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:30:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14292482-vmss000000-472fv\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-14292482-vmss000000-472fv has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T10:31:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dmqmp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-dmqmp has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:11:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-23980774-vmss000000-slpqz\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-23980774-vmss000000-slpqz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:11:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-lzv6r\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-lzv6r has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:11:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-qbqjf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-qbqjf has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:12:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"swift-recorder\", pod=\"swift-recorder-6f6gx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod swift-recorder/swift-recorder-6f6gx has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","signature_id":"6e8b49a5caf85ad1a9ceb24783ca3a6237c65bb0981b2afa932bd94b35ce4433","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","signature_id":"b224cee0c66252329215d260009da282cdb5e711becf7fce204c7d1cc07cca6f","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","signature_id":"74767212334f915bb5336f7d2acfaee3ea2d2733754d6e36e4f882ac6ca14cd5","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108258997998981120","occurred_at":"2026-10-08T18:11:29Z","signature_id":"da0348e89e81da96d29b328c4681e84b359b3ea85a39dbf681fbeba48167e0dd","pr_number":7433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107961335642853376","occurred_at":"2026-10-07T22:28:41Z","signature_id":"f3afea088c71927d2827ae8a8594776068254cc9904aa88bdae7db71ce3d503a","pr_number":7305,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647281405628416","occurred_at":"2026-10-07T01:40:44Z","signature_id":"9ee55a303c8e841323cc685a7ec3f29ef79b76512b54804abd0bdb38ccfc1218","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.9292604501607717,"anchor_occurrences":6,"after_last_push_count":3,"trend_counts":[2,1,2,3,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-10-05"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-09T15:02:04Z\n  Ended:       2026-10-09T15:12:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4897024-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T14:26:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j5137536-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T03:29:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j9299456-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","signature_id":"e01049157fc58f922b04b07385b9c89d671967d9a7a5d5a68d2d2c559ee1a0da","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108552522435137536","occurred_at":"2026-10-09T13:37:50Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108385613169299456","occurred_at":"2026-10-09T02:34:36Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":7394,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","occurred_at":"2026-10-08T14:14:35Z","signature_id":"22ea0c3803bb98326df9cbf23de00a80472321a99d7d574f14fa44cb6663b005","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","occurred_at":"2026-10-08T13:42:51Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":7427,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647281405628416","occurred_at":"2026-10-07T01:40:44Z","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.9292604501607717,"anchor_occurrences":6,"after_last_push_count":6,"also_in":["PROD"],"trend_counts":[0,1,0,2,0,1,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":27,"prior_last_seen_at":"2026-10-11T19:48:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23913a80e730\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23913a80e730\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x23a94d2a72e0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"f29a7407248c09d9f2d63fc02a535acb501328e35c4bfacc81f04e0e98c0e854","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660658009116672","occurred_at":"2026-10-12T15:00:29Z","signature_id":"f0b4f980a6b881fe20a87e1354e5792bdbf23c290ec125adfa5d9d993ae815ea","pr_number":7467,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109370651696435200","occurred_at":"2026-10-11T19:48:48Z","signature_id":"0942b9f98992dfeb8e8e398ab3ba99f81052ee6ffa88dc880de4c1cdafa43b86","pr_number":7457,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","occurred_at":"2026-10-09T06:02:56Z","signature_id":"c750fcad772bcae880a5d3b350c926fbfa2d24755d777353cee4665f043e8937","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","occurred_at":"2026-10-09T02:04:14Z","signature_id":"dd5e9783b1066a92e3ba3b8da9f3bde91680259100a31f4f4dc4745788500224","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107680144293367808","occurred_at":"2026-10-07T03:51:19Z","signature_id":"d1dfb5d720d4e924abf2ec1c610faee1a9a11cd6f9af55462d2c0c27633e01df","pr_number":7097,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":5,"after_last_push_count":3,"trend_counts":[0,3,0,2,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-09T21:48:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3475813fe160\u003e: \n    DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3475813fe0e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3475813fe160\u003e: \n    DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3475813fe0e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x247e648e4440\u003e: \n    DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x247e648e4420\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x247e648e4440\u003e: \n    DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x247e648e4420\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-ff6jdg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d44c09e520\u003e: \n    DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x17d44c09e500\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-ff6jdg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d44c09e520\u003e: \n    DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x17d44c09e500\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108676067857272832","occurred_at":"2026-10-09T21:48:46Z","signature_id":"133758dc096cedae43ff2559d3a74348d74fa3017fff8b919bd6d528e1c03295","pr_number":7165,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648873588166656","occurred_at":"2026-10-09T20:00:42Z","signature_id":"99c7ef686eed46a4a71bdbed72621290bd7f8e8a9969f168d079f9557b9819e9","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7417/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107967978476670976","occurred_at":"2026-10-07T22:55:04Z","signature_id":"117275c46ce91f05ea60621ea4c13a1b949e2cbb5298e2ee95b6e289338fe062","pr_number":7417,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"d617fd03098208e530c7ebbec73454d1787eb1156909d4b1d261a619471781dc","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107717187903950848","occurred_at":"2026-10-07T06:18:31Z","signature_id":"c0b3902507c3f57210df372dcd211846b078fe3f061059eb55d1e7f21f3822f6","pr_number":7402,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,1,0,2,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-09T20:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n    DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n    DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-j7bk7p\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x76fa41aa1e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x76fa41aa1a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-j7bk7p\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x76fa41aa1e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x76fa41aa1a0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109549126109630464","occurred_at":"2026-10-12T07:37:59Z","signature_id":"ebfef50e4f33847feb7608db52e9f990adccd34b42505d1aca31667f15925d12","pr_number":7237,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109538389626720256","occurred_at":"2026-10-12T06:55:20Z","signature_id":"4673534f224bf242b8df8885a07c9c85c02ab3a626129d3db6b153550df27ba6","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648932484583424","occurred_at":"2026-10-09T20:00:53Z","signature_id":"1bd05fb2c90549773c1175fd56546ae62c47d903fe566a61ea1dc83c30a119d8","pr_number":7227,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","signature_id":"997f39d2360594b30e790410b45db1739cf0adf16512701d1c5d310dca5ad763","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"b2b88a83521cee771cd93ba8e8c7482cb474d79b4714e2d4c669fa1ea2c77764","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":5,"after_last_push_count":7,"also_in":["PROD"],"trend_counts":[3,0,1,1,0,0,3,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":29,"prior_last_seen_at":"2026-10-09T16:02:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe01e7de22c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2x2v84z9v765\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\",\n      \"name\": \"3d382676-ff60-4e1f-babd-b92045c21565\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:22:34.907448575Z\",\n      \"endTime\": \"2026-10-12T19:51:42.771361832Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n            correlation ID: 7441ce1a-f7e5-42c1-bf5b-99ce537aeb48\n            client request ID: fed3af2b-11b1-43b5-8577-26857bd2a8c1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe01e7de22c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2x2v84z9v765\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\",\n      \"name\": \"3d382676-ff60-4e1f-babd-b92045c21565\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:22:34.907448575Z\",\n      \"endTime\": \"2026-10-12T19:51:42.771361832Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3d382676-ff60-4e1f-babd-b92045c21565\n            correlation ID: 7441ce1a-f7e5-42c1-bf5b-99ce537aeb48\n            client request ID: fed3af2b-11b1-43b5-8577-26857bd2a8c1\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x329d818e85e0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7hcwvl89hlds\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\",\n      \"name\": \"af485567-16d3-466b-8cef-6a0e9bce52fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:42:57.954228784Z\",\n      \"endTime\": \"2026-10-12T13:11:58.34288805Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n            correlation ID: 70b5741c-ea80-4b9a-9e33-ac7a70af1ce7\n            client request ID: f176988a-7ecb-4ecf-8063-4ee5d27bb712\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x329d818e85e0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7hcwvl89hlds\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\",\n      \"name\": \"af485567-16d3-466b-8cef-6a0e9bce52fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:42:57.954228784Z\",\n      \"endTime\": \"2026-10-12T13:11:58.34288805Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/af485567-16d3-466b-8cef-6a0e9bce52fa\n            correlation ID: 70b5741c-ea80-4b9a-9e33-ac7a70af1ce7\n            client request ID: f176988a-7ecb-4ecf-8063-4ee5d27bb712\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d2ba8a26f40\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-tmxkg7m275vm\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\",\n      \"name\": \"06ab8b07-36db-4738-9b5d-4d16092f8301\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T09:26:40.450447729Z\",\n      \"endTime\": \"2026-10-12T09:55:40.068785838Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n            correlation ID: c5b7b9a4-edc0-484e-9068-a103f50c2ab8\n            client request ID: d0e69db4-dd9a-407e-93a4-3206c81ac5a6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d2ba8a26f40\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-tmxkg7m275vm\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\",\n      \"name\": \"06ab8b07-36db-4738-9b5d-4d16092f8301\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T09:26:40.450447729Z\",\n      \"endTime\": \"2026-10-12T09:55:40.068785838Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06ab8b07-36db-4738-9b5d-4d16092f8301\n            correlation ID: c5b7b9a4-edc0-484e-9068-a103f50c2ab8\n            client request ID: d0e69db4-dd9a-407e-93a4-3206c81ac5a6\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"ba9594d183dcf679de486f3ef983cb99f7de22b5534227c654d1211082cc3ca0","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"614ca0e1ea00ef1b4414e6ab8b3a11a8b2da5132a3da64d8b9faf88c1203affd","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"026e1543f757852d519080f003f4d7ebec18027795d5835ee8f594400d820c5e","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108589017825021952","occurred_at":"2026-10-09T16:02:30Z","signature_id":"7ac519174700b8af8b38e72050ab85056e2459914e3de2dee5a59bdd798501b7","pr_number":7227,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"09cb0962695c90bbcb8aa045ebb144f397afbd0455023f7fea0160df7e60cafc","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,0,2,1,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-09T09:35:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":5}],"full_error_samples":["alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6482688: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6482688/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"ResourceGroupNotFound\",\n    \"message\": \"Resource group 'hcp-underlay-ci01-j6482688' could not be found.\"\n  }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"7dde37c662e4768f610ecb85052e99f87f88976e1e543acbe408c61888caa61d","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"3223dd11b35ca881a6905f2213555998ec56974987e667be15a6afbf19cd4f19","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","occurred_at":"2026-10-09T09:35:37Z","signature_id":"a5fd6e34535fa450e59098cc1740b0f19b4969d471d45588dd16e29233350a75","pr_number":7226,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","signature_id":"a13b64d94d488082a4a9ffd8fdeb35b5d5d0a71c93381c3fedbe0bba79b488d1","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","signature_id":"2859658063bed829d162181cb3d76e39201e6bf4d59aca2d6c51092fd846a233","pr_number":7226,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,3,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T21:30:33Z\n  Ended:       2026-10-12T22:07:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j6043392-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s (management cluster ci01-j6043392-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T19:50:18Z\n  Ended:       2026-10-12T20:25:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2503936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tddoajh9tanmcldflt6gegfe8au9a4r-a8g2t7k0z6k5s9l\", region=\"centralus\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tddoajh9tanmcldflt6gegfe8au9a4r-a8g2t7k0z6k5s9l (management cluster ci01-j2503936-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T19:45:35Z\n  Ended:       2026-10-12T20:14:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j5649024-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tddn1vgtlc9nve2f6icfn7fqenmjbmm-x2t6l2r1k1q6d2z\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tddn1vgtlc9nve2f6icfn7fqenmjbmm-x2t6l2r1k1q6d2z (management cluster ci01-j5649024-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"0c18256aec4487d73b73645b8e1ded1390ad9218643d5179523cb223b9edd0b5","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109706138382503936","occurred_at":"2026-10-12T18:01:54Z","signature_id":"1e41fec641cef36701893933a54d28e9a3e41519280905104056febcd0e7dcfb","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109705158475649024","occurred_at":"2026-10-12T17:58:00Z","signature_id":"1de0d91a762f7a24003e4b3846ddd93be7eb53f9d5a9baa369ea319d275302ba","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9fb80e317ba17a110379c8169e33f721a9e1b46fd5cbd72a1477ce7cdef9b69a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"947f42338d6e84abb9bb52e5b52d433427544ab1231ca5eddc6f0f75313afdfc","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE advances a control plane to an SRE-pinned version","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":1.607717041800643,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,5,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE advances a control plane to an SRE-pinned version","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-zx5s4v\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-zx5s4v\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.009s.\ncluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.009s.\ncluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36\nExpected\n    \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n    \u003cstring\u003e: 4.21.36"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","signature_id":"8ec28f71b625b2c11ea506fd0748031d3bf5d074d3279546ef6bfc6cf0d20aaf","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","signature_id":"22c47845096418cb71e4fa35973233fa4b21f339d382954b3aee25b82c29cb84","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","signature_id":"74adfdd35c7ce8eb88d441864d6b1e0ec991a2d255378a9ffce5b6cd5cf8549e","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","signature_id":"b12b641e87320622b3d715665e73509b0e2928b9b8bc71c938cb2b7200e48d3e","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600098458537984","occurred_at":"2026-10-12T11:00:15Z","signature_id":"f0121469ca4f42ea3b4cc9ff22e2d69e6a7cdd34679fb94a79eb413022a0a715","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.2861736334405145,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,1,0,4,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-09T11:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1d94d9c80a8\u003e: \n    resource group \"autoscaling-cluster-tj5rdtq5zbg6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tj5rdtq5zbg6\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\",\n      \"name\": \"962a68bc-b321-487b-b276-789b2307a373\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:58:48.91579633Z\",\n      \"endTime\": \"2026-10-09T12:22:56.421671364Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1d94d9c80a8\u003e: \n    resource group \"autoscaling-cluster-tj5rdtq5zbg6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tj5rdtq5zbg6\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\",\n      \"name\": \"962a68bc-b321-487b-b276-789b2307a373\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:58:48.91579633Z\",\n      \"endTime\": \"2026-10-09T12:22:56.421671364Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3346ee20e4b0\u003e: \n    resource group \"feature-aggregation-vk8bg8lddcq2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vk8bg8lddcq2\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\",\n      \"name\": \"4724c843-31f5-4e70-a53a-a9dc39567964\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:05:11.356025969Z\",\n      \"endTime\": \"2026-10-09T12:29:15.890678807Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3346ee20e4b0\u003e: \n    resource group \"feature-aggregation-vk8bg8lddcq2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vk8bg8lddcq2\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\",\n      \"name\": \"4724c843-31f5-4e70-a53a-a9dc39567964\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:05:11.356025969Z\",\n      \"endTime\": \"2026-10-09T12:29:15.890678807Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x10e954cac0c0\u003e: \n    resource group \"fips-enabled-g648xvh6nzj9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-g648xvh6nzj9\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n      \"name\": \"6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:52:00.540461838Z\",\n      \"endTime\": \"2026-10-09T12:16:05.840029374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x10e954cac0c0\u003e: \n    resource group \"fips-enabled-g648xvh6nzj9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-g648xvh6nzj9\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n      \"name\": \"6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:52:00.540461838Z\",\n      \"endTime\": \"2026-10-09T12:16:05.840029374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"6c94a10ce89058dfd74a3fdd386049b899cca67c20e5c87b6187a4e4e622cee2","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"ca2f7f404a7338877b8aee12b28b784744b9901ec82342acf628569b744e7de9","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"d58e1b5ab93a98812a74d2fb9e1201c1960008bc7c934bc40a4972c4e065df59","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"d18215fee9182d6fd0dffacde080da49fadf92738df8b9767a5a7d2cf9cc4833","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"8c20bdbff176792aa32d1f70b7055efc5adcd7e98259606ec8b663cd51ef5fe0","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.2861736334405145,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,3,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-10T11:45:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n      \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n            correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n            client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n      \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n            correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n            client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n      \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n            correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n            client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n      \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n            correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n            client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n      \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n            correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n            client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n      \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n            correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n            client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109749870842613760","occurred_at":"2026-10-12T20:55:41Z","signature_id":"69699f183ac245894ec15c2257ce86fb84bc1f5bed2cf85b38d4c739dedbd9d1","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109742847518314496","occurred_at":"2026-10-12T20:27:46Z","signature_id":"d4b16f343401a26bebbb19567d2aeea68fce56219f43f4bb17023f4763f88712","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"f06a1fb8a7fb87970f3aa02a8622a506c32fd712c86f231325a5e5fce4359407","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108886756110635008","occurred_at":"2026-10-10T11:45:56Z","signature_id":"9df37f22bf11ea31242e52f951190e90a4071cb737774ac577107261d4da179e","pr_number":7374,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":1.2861736334405145,"anchor_occurrences":4,"after_last_push_count":12,"trend_counts":[2,0,1,0,0,0,3,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":52,"prior_last_seen_at":"2026-10-08T04:44:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T19:53:49Z\n  Ended:       2026-10-12T20:09:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cdb52\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T19:53:49Z\n  Ended:       2026-10-12T20:13:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2661632-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2661632/providers/microsoft.monitor/accounts/hcps-j2661632\", namespace=\"ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cdb52\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tdeaut6dntudqr0or9vd7b3n8j9vs0m-p6q2s1e0g0h0f7y/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T13:06:14Z\n  Ended:       2026-10-12T13:34:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5f8cm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T13:06:14Z\n  Ended:       2026-10-12T13:29:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5598976-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.236:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5598976/providers/microsoft.monitor/accounts/hcps-j5598976\", namespace=\"ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5f8cm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2td8fkj2kvk4mcasoetkg4c7kcrid62p-cilium-cl/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T09:49:03Z\n  Ended:       2026-10-12T10:13:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4hsb2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T09:49:03Z\n  Ended:       2026-10-12T10:17:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9143424-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.60:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9143424/providers/microsoft.monitor/accounts/hcps-j9143424\", namespace=\"ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4hsb2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2td5jk909ajmhn3gj3ep0upql88758lp-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"01b14f16705c690af3cc17499286b7047bac4716208b40b6f4d834caa2ce6ebc","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"59230409e21419e206546b3eed7180a2b2d9241870b6db3da89dea2764805770","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"cfbb3abf844cd53f28b64e3663605f410242ad10ab7e9f569183150881111e11","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"85682e6aa72ffd8e2527c0a0e4f27430fd467da3643a75af116ad2157337fe40","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.9646302250803859,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-28","2026-10-05"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x151218b082b8\u003e: \n    resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x151216d07800\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x151218b082b8\u003e: \n    resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x151216d07800\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x15121775bc50\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x213e82b661e0\u003e: \n    resource group \"rg-cluster-back-version-s9llrx7qtffw\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x213e826ac820\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-s9llrx7qtffw\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x213e826ac800\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x213e83e111d0\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x213e82b661e0\u003e: \n    resource group \"rg-cluster-back-version-s9llrx7qtffw\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x213e826ac820\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-s9llrx7qtffw\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x213e826ac800\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x213e83e111d0\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3c28962ca078\u003e: \n    resource group \"rg-cluster-back-version-kcpxjfv4fpw8\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3c289468e340\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-kcpxjfv4fpw8\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x3c289468e320\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x3c2894a68660\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3c28962ca078\u003e: \n    resource group \"rg-cluster-back-version-kcpxjfv4fpw8\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3c289468e340\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-kcpxjfv4fpw8\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x3c289468e320\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x3c2894a68660\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"b46b21369678c0c798aca134326c4166ed0c77d047a753de7c68744c3f55b704","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4d8dc6809a1434d6bdd49c263f770a407adc719dc2f0c6e5c6248a598c1ba4ad","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ad853bd0242f92cb0e2af68ec14f11bc3c9f69d37f2035d60a8204f74aa62438","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.9646302250803859,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[1,0,1,0,0,1,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-11T23:05:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23adf9021e40\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-92gd76g6d97g\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n            correlation ID: 349f9734-994e-46db-888e-e1caf77e8fb4\n            client request ID: 46549edd-5c88-4d3d-b17b-df9c20ad793d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23adf9021e40\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-92gd76g6d97g\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n            correlation ID: 349f9734-994e-46db-888e-e1caf77e8fb4\n            client request ID: 46549edd-5c88-4d3d-b17b-df9c20ad793d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n            correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n            client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zhszcgnzg7zb/cluster-candidate-4-22-gwkf7b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17457dc5e0e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-gwkf7b: failed waiting for cluster=\"cluster-candidate-4-22-gwkf7b\" in resourcegroup=\"rg-candidate-4-22-zhszcgnzg7zb\" to finish creating: GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48785925-6e9d-4c48-aae6-599ebf8c4425\n            correlation ID: e870b595-733c-4e50-b159-c06646eb16ba\n            client request ID: 7c347d72-8099-4796-893d-59762507c230\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1dd12c4c3a80\u003e: \n    failed to create NodePool np-ft6nng: failed waiting for nodepool=\"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\" in resourcegroup=\"rg-candidate-4-22-j25t2jm5j66l\" to finish creating: GET https://rp.j9390848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1dd12c4c3a80\u003e: \n    failed to create NodePool np-ft6nng: failed waiting for nodepool=\"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\" in resourcegroup=\"rg-candidate-4-22-j25t2jm5j66l\" to finish creating: GET https://rp.j9390848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","signature_id":"9b6a19c4bbd8e0fee198397011d61bfa3837fe3c4ea1afea5bc1a3c2f92f5308","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109420238498959360","occurred_at":"2026-10-11T23:05:50Z","signature_id":"b2888b7c2e1a7a0a79a19f9ced813090e269f2019ace640a3be1b766564988b0","pr_number":7424,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108217820029390848","occurred_at":"2026-10-08T15:27:51Z","signature_id":"a9e7d98d59a4178bf41e7b5661530ffe552aba4c92d51d87d0a3a1f5a9752100","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.9646302250803859,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-08T07:02:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xef3ce514120\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xef3ce514100\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0xef3ce5140e0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xef3ce514120\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xef3ce514100\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0xef3ce5140e0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d63e63ce540\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d63e63ce520\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x1d63e63ce4e0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d63e63ce540\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1d63e63ce520\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x1d63e63ce4e0\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x5336d4f4280\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x5336d4f4220\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x5336d4f4180\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x5336d4f4280\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x5336d4f4220\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x5336d4f4180\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","occurred_at":"2026-10-08T07:02:22Z","signature_id":"06adbd5f174da4a722aae0d056ec2b5a7c54b016aa9e689d5cee91e57df2195d","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","signature_id":"dc991bbf770e80e3e153a619922727dc6f068b8152a0521def1711d4a491b244","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107970662109810688","occurred_at":"2026-10-07T23:05:44Z","signature_id":"9af334222a9a1079a318e8007182007684788c07d1bd75371b6aa376ab0cd4f3","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,1,0,2,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-09T11:01:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x14de9654a318\u003e: \n    resource group \"arm64-vm-cluster-bbmplmj7xnjh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bbmplmj7xnjh\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n      \"name\": \"8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:10:18.071494213Z\",\n      \"endTime\": \"2026-10-09T12:34:22.504221789Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x14de9654a318\u003e: \n    resource group \"arm64-vm-cluster-bbmplmj7xnjh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bbmplmj7xnjh\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n      \"name\": \"8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:10:18.071494213Z\",\n      \"endTime\": \"2026-10-09T12:34:22.504221789Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xdad5eeb8138\u003e: \n    resource group \"external-auth-cluster-cmj7tlkjf2f4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cmj7tlkjf2f4\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\",\n      \"name\": \"a7900d0f-9832-43df-824c-157a3931804e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:11:18.997805773Z\",\n      \"endTime\": \"2026-10-09T12:35:21.856574986Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xdad5eeb8138\u003e: \n    resource group \"external-auth-cluster-cmj7tlkjf2f4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cmj7tlkjf2f4\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\",\n      \"name\": \"a7900d0f-9832-43df-824c-157a3931804e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:11:18.997805773Z\",\n      \"endTime\": \"2026-10-09T12:35:21.856574986Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3ed0c46420d8\u003e: \n    resource group \"rg-negative-tests-z8rjxzl5wzq5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-z8rjxzl5wzq5\" to finish deleting: GET https://rp.j8408192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"name\": \"92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:31:10.664805555Z\",\n      \"endTime\": \"2026-10-07T20:55:12.515323239Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3ed0c46420d8\u003e: \n    resource group \"rg-negative-tests-z8rjxzl5wzq5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-z8rjxzl5wzq5\" to finish deleting: GET https://rp.j8408192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"name\": \"92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:31:10.664805555Z\",\n      \"endTime\": \"2026-10-07T20:55:12.515323239Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","signature_id":"4806f8d967c169f14897e23c9636721ed45b628d0224580a25d7df612eeaf06d","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","signature_id":"bbfbd4cef85a808bf51623d1118bb2a4b11c9f5723a7f47496489be8ca80a6ce","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"0ecb42980a83f2bed43b3f38c65950d796ceed1be274e4dbc2e1013fe3707753","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":3,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,2,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x31481464c780\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vxxh5xhm4hm8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x314814c9ae20\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x31481464c780\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vxxh5xhm4hm8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x314814c9ae20\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x13eb0e2191a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-bkn7fgqj44mp\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x13eb0e00ddf0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x13eb0e2191a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-bkn7fgqj44mp\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x13eb0e00ddf0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x7f2f57b95f0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hwcbq22lzw9l\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n            },\n            \u003c*fmt.wrapErrors | 0x7f2f57b95c0\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n                        s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x7f2f57b95f0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hwcbq22lzw9l\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n            },\n            \u003c*fmt.wrapErrors | 0x7f2f57b95c0\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n                        s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"bec2ded38a9a866918a5ba9ad1e9a064412687841aabfca8c59e91dc8567ad9c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"8e8ca9f4e7208efb5d4591e5e48ad01e865e6837bb0c14b95fb6aaa7aa5f6c67","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"aa8d5fde17c317b9352aca696c164c9e194b56fe7f66ffe5c727b348eda15428","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8af94ff77be9280ec3040c21d82445dd604d4b00fdf26e371d68448461bff48d","failure_pattern":"Command failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-\u003cid\u003e/pipeline@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/ci-op-\u003cid\u003e/pipeline:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-jhx30i3q/pipeline@sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 arohcpsvcdev.azurecr.io/ci-op-jhx30i3q/pipeline:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 --from-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json --to-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet/global/mirror-image","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-07T01:41:52Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet/global/mirror-image","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.KubeApplier/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-10-07T02:30:15.367Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e fleet.image.registry \u003cnil\u003e}:{\u003cnil\u003e fleet.image.repository \u003cnil\u003e}@{\u003cnil\u003e fleet.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-07T02:32:48.216Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build11.ci.openshift.org\\nSaved credentials for registry.build11.ci.openshift.org into /tmp/tmp.lx56KPpMDi/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build11.ci.openshift.org/ci-op-jhx30i3q/pipeline@sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 to arohcpsvcdev.azurecr.io/ci-op-jhx30i3q/pipeline:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604.\\nThe image will still be available under it's original digest sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 in the target registry.\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unavailable: service unavailable: health check failed: please see /debug/health\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unavailable: service unavailable: health check failed: please see /debug/health\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-jhx30i3q/pipeline@sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 arohcpsvcdev.azurecr.io/ci-op-jhx30i3q/pipeline:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 --from-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json --to-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json\\n exit status 1\"","time=2026-10-07T02:34:43.899Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.KubeApplier resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e kubeApplier.image.registry \u003cnil\u003e}:{\u003cnil\u003e kubeApplier.image.repository \u003cnil\u003e}@{\u003cnil\u003e kubeApplier.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-07T02:35:51.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.KubeApplier resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build11.ci.openshift.org\\nSaved credentials for registry.build11.ci.openshift.org into /tmp/tmp.BaWVNiNbL6/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build11.ci.openshift.org/ci-op-n30v64rp/pipeline@sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 to arohcpsvcdev.azurecr.io/ci-op-n30v64rp/pipeline:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894.\\nThe image will still be available under it's original digest sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 in the target registry.\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-n30v64rp/pipeline@sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 arohcpsvcdev.azurecr.io/ci-op-n30v64rp/pipeline:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 --from-registry-config /tmp/tmp.BaWVNiNbL6/containers/auth.json --to-registry-config /tmp/tmp.BaWVNiNbL6/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647565049630720","occurred_at":"2026-10-07T01:41:52Z","signature_id":"2d70df3f3655f8e6d61803842ff1a711d8b43be234f53bc324901afc6271aab3","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647281405628416","occurred_at":"2026-10-07T01:40:44Z","signature_id":"16290eb32430f166cec00354029ad2ead97a9b7e00cd7ae123762fc3b2a7ef4a","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10bf87371b00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w96jh79xf66q\" to finish creating: GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\",\n      \"name\": \"1173880a-8bf8-448d-a470-259a36f4e033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:18:06.878392832Z\",\n      \"endTime\": \"2026-10-12T16:47:15.056749067Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n            correlation ID: 685034b4-f64b-4ad2-9968-7845cc5d96c5\n            client request ID: 20bd3fae-18f5-45e3-8cd8-b2f1ea3f8020\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0x10bf87371b00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w96jh79xf66q\" to finish creating: GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\",\n      \"name\": \"1173880a-8bf8-448d-a470-259a36f4e033\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:18:06.878392832Z\",\n      \"endTime\": \"2026-10-12T16:47:15.056749067Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n            correlation ID: 685034b4-f64b-4ad2-9968-7845cc5d96c5\n            client request ID: 20bd3fae-18f5-45e3-8cd8-b2f1ea3f8020\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11fa8d633b80\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dmqcn9\" in resourcegroup=\"rg-np-noedge-dmqcn9-8bfmwp2gm92p\" to finish creating: GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n      \"name\": \"5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T03:07:24.415643123Z\",\n      \"endTime\": \"2026-10-12T03:36:29.188004519Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n            correlation ID: 72fd1a70-e0c7-48fa-aa70-d0f6edab139d\n            client request ID: 04d73669-7402-48f6-a103-fc2cac633166\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x11fa8d633b80\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dmqcn9\" in resourcegroup=\"rg-np-noedge-dmqcn9-8bfmwp2gm92p\" to finish creating: GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n      \"name\": \"5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T03:07:24.415643123Z\",\n      \"endTime\": \"2026-10-12T03:36:29.188004519Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n            correlation ID: 72fd1a70-e0c7-48fa-aa70-d0f6edab139d\n            client request ID: 04d73669-7402-48f6-a103-fc2cac633166\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","signature_id":"222aacc360be62a53a3a25918371654d8976e4d02348bfbb0f85f02afd94062c","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109460696247832576","occurred_at":"2026-10-12T01:46:36Z","signature_id":"8363386ebe99557c4d0a5f2ab5fc61129ce9cc8c82392c35b5213d30f26dde65","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3990eaeace20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9d78f6chpq4q\" to finish creating: GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n      \"name\": \"75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T22:04:42.193524873Z\",\n      \"endTime\": \"2026-10-12T22:33:43.052616409Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n            correlation ID: e27c7927-08d3-45ea-b9cc-9889d4e15592\n            client request ID: e73a526e-7c8b-4130-ac0c-beedee83ba98\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3990eaeace20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9d78f6chpq4q\" to finish creating: GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n      \"name\": \"75420ed5-ee81-4901-8aca-fe33d4e52f8f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T22:04:42.193524873Z\",\n      \"endTime\": \"2026-10-12T22:33:43.052616409Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-9d78f6chpq4q--managed/providers/Microsoft.Compute/virtualMachines/z7o2s9y6n3g5u0s-np-1-mfg5x-rg5mg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0512512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75420ed5-ee81-4901-8aca-fe33d4e52f8f\n            correlation ID: e27c7927-08d3-45ea-b9cc-9889d4e15592\n            client request ID: e73a526e-7c8b-4130-ac0c-beedee83ba98\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d62ef202100\u003e: \n    failed to create NodePool np-d4z57m: failed waiting for nodepool=\"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\" in resourcegroup=\"rg-hs-pre--zplpn9jd4vzj\" to finish creating: GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\",\n      \"name\": \"2867897e-0970-460c-81d6-4c5fa1ec8583\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:39:38.582635827Z\",\n      \"endTime\": \"2026-10-12T20:08:43.787852166Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n            correlation ID: 56d0624b-f6fa-49bb-b96e-3fa32af8a523\n            client request ID: c9d4e044-239c-4fd2-8127-3707455ec72b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d62ef202100\u003e: \n    failed to create NodePool np-d4z57m: failed waiting for nodepool=\"np-d4z57m\" for cluster \"cluster-hs-pre-d4z57m\" in resourcegroup=\"rg-hs-pre--zplpn9jd4vzj\" to finish creating: GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\",\n      \"name\": \"2867897e-0970-460c-81d6-4c5fa1ec8583\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:39:38.582635827Z\",\n      \"endTime\": \"2026-10-12T20:08:43.787852166Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zplpn9jd4vzj-d4z57m--managed/providers/Microsoft.Compute/virtualMachines/f9u7j8t0x8a4t1u-np-d4z57m-s74xc-h7jlv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2582016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2867897e-0970-460c-81d6-4c5fa1ec8583\n            correlation ID: 56d0624b-f6fa-49bb-b96e-3fa32af8a523\n            client request ID: c9d4e044-239c-4fd2-8127-3707455ec72b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","signature_id":"e7d1ae872e2d1e1f299e0950e3c7121739a7ca3253cb34361fe8fe1de4b27d3a","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109709852522582016","occurred_at":"2026-10-12T18:16:39Z","signature_id":"f07f78c84cfe8e58c2511543b04e0ca7a20c652e502f4d4a311097ea89b1549b","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n      \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n            correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n            client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n      \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n            correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n            client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bbf606b9160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-frt9kkhsnfm2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n      \"name\": \"75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T11:36:28.504164951Z\",\n      \"endTime\": \"2026-10-12T12:05:35.482369546Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qmq6h-h7gmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qmq6h-h7gmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n            correlation ID: d89bc198-69ae-4651-acfe-b566b134eb9a\n            client request ID: d8b88a68-6fe2-4293-891c-8ad5621c82b5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bbf606b9160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-frt9kkhsnfm2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n      \"name\": \"75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T11:36:28.504164951Z\",\n      \"endTime\": \"2026-10-12T12:05:35.482369546Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qmq6h-h7gmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qmq6h-h7gmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n            correlation ID: d89bc198-69ae-4651-acfe-b566b134eb9a\n            client request ID: d8b88a68-6fe2-4293-891c-8ad5621c82b5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"0b8a3b5437beafe0105aa62d86153fdc67ea6aaf34efdb9197853845e0812245","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"810897085d9afd12e972e9199e65575a09acca1a53e133ff2eb94e81c5444daf","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-08T15:54:02Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":2}],"full_error_samples":["time=2026-10-08T16:19:05.365Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T16:30:17.342Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T16:19:05.375275155Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T16:19:08.114395254Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T16:19:08.462375095Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335389178Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6F2FEEE9F3FAC221\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:19:14.8133695Z\\\",\\\"duration\\\":\\\"PT9M5.9123598S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335532727Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"98034B352C012EE7\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT4.7536001S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.33558093Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DE129F3B0C485FAF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT1M38.1712414S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335599002Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B325BEA3E7A07E7A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT1M56.0129258S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335620623Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D7A0CEADFD719B73\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT3.9567105S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335657835Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7F61B179B809D260\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:19:14.8133695Z\\\",\\\"duration\\\":\\\"PT9M0.9591654S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335709239Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584101305363180932\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"11m11.960448525s\\\"}\\n exit status 1\"","time=2026-10-08T16:14:31.581Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T16:22:39.946Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T16:14:31.590606721Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T16:14:33.50353792Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T16:14:33.891298669Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944584337Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"241855BF9A0D266F\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT1M38.1681956S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944653829Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5B81E75ADA721336\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.9210349S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.94468287Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"0A9A6CB955E8A5E9\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:14:38.0622167Z\\\",\\\"duration\\\":\\\"PT7M50.9531763S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944693871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"127F3C3755EDC782\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.2491544S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944712361Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1AF780C01A639DF0\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.2335315S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944730542Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2DBAB0983C574AF5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:14:38.0622167Z\\\",\\\"duration\\\":\\\"PT5M54.4662985S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944752483Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584101308111130568\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"8m8.354161621s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108224410405048320","occurred_at":"2026-10-08T15:54:02Z","signature_id":"664aa8f1aeb2650939331a96787558c2e7340b55e45ef2847dd30061d3213023","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108222712823418880","occurred_at":"2026-10-08T15:47:18Z","signature_id":"94dd4a2e46776ab0e7c5e9b444cb5e7c4246df9918134350ba21a74752e8098b","pr_number":7364,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,1,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-28","2026-10-05"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-08T07:10:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n      \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n            correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n            client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n      \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n            correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n            client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18aecfdf0aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dq2w5s6mnvnb\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n      \"name\": \"5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T08:17:53.86568203Z\",\n      \"endTime\": \"2026-10-08T08:18:02.050800903Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18aecfdf0aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dq2w5s6mnvnb\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n      \"name\": \"5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T08:17:53.86568203Z\",\n      \"endTime\": \"2026-10-08T08:18:02.050800903Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109655959428141056","occurred_at":"2026-10-12T14:42:30Z","signature_id":"0b00e0911be997c70989e7dd3678d4edc17e5aca2336e84d6fe6cf814fadeb28","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","signature_id":"99a86419200aac49952e3148c95bd65fe8b3fbe3e68a7b83763b8f9aa7967171","pr_number":7412,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0cce093122447f5c3b6f99be8e0de298d1fbee72db77c1703737ae037ec0882b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-08T08:20:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xd2898b7c180\u003e: \n    resource group \"private-kas-f55dwxz4lh8f\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-f55dwxz4lh8f\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n      \"name\": \"f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T10:06:03.074876755Z\",\n      \"endTime\": \"2026-10-08T10:30:10.535297041Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tahe6f6jvv1r5au11pgmqub5ah0te26 still exists (deletion dispatched at 2026-10-08T10:06:03Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xd2898b7c180\u003e: \n    resource group \"private-kas-f55dwxz4lh8f\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-f55dwxz4lh8f\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n      \"name\": \"f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T10:06:03.074876755Z\",\n      \"endTime\": \"2026-10-08T10:30:10.535297041Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tahe6f6jvv1r5au11pgmqub5ah0te26 still exists (deletion dispatched at 2026-10-08T10:06:03Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2183e1e34210\u003e: \n    resource group \"private-ingress-76lkp46mpc8d\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-76lkp46mpc8d\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"name\": \"1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:03:40.469898232Z\",\n      \"endTime\": \"2026-10-07T18:27:41.300163086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cd0prbd7pkt4n50qh2rsnhp0bjo4 still exists (deletion dispatched at 2026-10-07T18:03:41Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2183e1e34210\u003e: \n    resource group \"private-ingress-76lkp46mpc8d\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-76lkp46mpc8d\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"name\": \"1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:03:40.469898232Z\",\n      \"endTime\": \"2026-10-07T18:27:41.300163086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cd0prbd7pkt4n50qh2rsnhp0bjo4 still exists (deletion dispatched at 2026-10-07T18:03:41Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","signature_id":"a341e9bf98d3c6591ee21f0e758f438f31604f2f35d4323eac348c53761f7bb0","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"ecbdfd8493553d96ba4723a52e827625dceadf0a4d4fcf8208229496ffbca565","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-09T11:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3c8ada036198\u003e: \n    resource group \"rg-candidate-4-22-7xk4vqr4kmpg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-68b4x4\" in resourcegroup=\"rg-candidate-4-22-7xk4vqr4kmpg\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n      \"name\": \"25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:01:27.302551031Z\",\n      \"endTime\": \"2026-10-09T12:25:36.409770296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kods8necs5a4oar2jo8ji1411tb still exists (deletion dispatched at 2026-10-09T12:01:28Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3c8ada036198\u003e: \n    resource group \"rg-candidate-4-22-7xk4vqr4kmpg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-68b4x4\" in resourcegroup=\"rg-candidate-4-22-7xk4vqr4kmpg\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n      \"name\": \"25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:01:27.302551031Z\",\n      \"endTime\": \"2026-10-09T12:25:36.409770296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kods8necs5a4oar2jo8ji1411tb still exists (deletion dispatched at 2026-10-09T12:01:28Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3172298ee060\u003e: \n    resource group \"customer-rg-85nx9txxtvjg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-85nx9txxtvjg\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\",\n      \"name\": \"c208ad27-e494-4933-aedb-2b311ea04677\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:49:09.861950907Z\",\n      \"endTime\": \"2026-10-09T12:13:16.248989406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb87377q9r4c48pr0vofden0ac6k3ii still exists (deletion dispatched at 2026-10-09T11:49:10Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3172298ee060\u003e: \n    resource group \"customer-rg-85nx9txxtvjg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-85nx9txxtvjg\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\",\n      \"name\": \"c208ad27-e494-4933-aedb-2b311ea04677\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T11:49:09.861950907Z\",\n      \"endTime\": \"2026-10-09T12:13:16.248989406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb87377q9r4c48pr0vofden0ac6k3ii still exists (deletion dispatched at 2026-10-09T11:49:10Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","signature_id":"cc670ab2e0f9fbf285a3decb00d2ef7f9ebf1bed8c73fe577bdc097fa7b42136","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","signature_id":"a607236403e65598e0d7a777b4d162c897ebc644542af7ce0b13c29ab958f2de","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[1,0,1,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-09T11:02:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2403780381b0\u003e: \n    resource group \"rg-candidate-4-22-2twmlnh2pntd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-97w8j7\" in resourcegroup=\"rg-candidate-4-22-2twmlnh2pntd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\",\n      \"name\": \"ec134b00-506f-4598-8f43-9039b6949cbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:19:56.634862294Z\",\n      \"endTime\": \"2026-10-09T12:44:04.410412622Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2403780381b0\u003e: \n    resource group \"rg-candidate-4-22-2twmlnh2pntd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-97w8j7\" in resourcegroup=\"rg-candidate-4-22-2twmlnh2pntd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\",\n      \"name\": \"ec134b00-506f-4598-8f43-9039b6949cbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T12:19:56.634862294Z\",\n      \"endTime\": \"2026-10-09T12:44:04.410412622Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1c054b45c180\u003e: \n    resource group \"rg-candidate-4-21-l7wm67p4w7mm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-pjnsh8\" in resourcegroup=\"rg-candidate-4-21-l7wm67p4w7mm\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\",\n      \"name\": \"de22b374-9b97-455f-b396-0c997a7b79ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:28:41.684717507Z\",\n      \"endTime\": \"2026-10-08T17:52:48.530098677Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1c054b45c180\u003e: \n    resource group \"rg-candidate-4-21-l7wm67p4w7mm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-pjnsh8\" in resourcegroup=\"rg-candidate-4-21-l7wm67p4w7mm\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\",\n      \"name\": \"de22b374-9b97-455f-b396-0c997a7b79ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:28:41.684717507Z\",\n      \"endTime\": \"2026-10-08T17:52:48.530098677Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","signature_id":"33c8fe82065035c70356f5c9aae13601f066c147beb33cc8c02dde9d53b03f2f","pr_number":7437,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","signature_id":"7ca525a80ef321179f5efc8d43f4c69d8ec239f2d58e9a339c8fd79d95ff3fb1","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f9989e753db1daf9717992966720fb7c20aebb56a0a3e0abe95adf30bb4255a7","failure_pattern":"Process did not finish before 15m0s timeout","search_query":"Process did not finish before 15m0s timeout","failed_at":"unknown","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"unknown","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:178\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 15m0s timeout\",\"severity\":\"error\",\"time\":\"2026-10-07T15:54:40Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:178\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 15m0s timeout\",\"severity\":\"error\",\"time\":\"2026-10-07T06:30:56Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"20a552c9ef65c7031e4db49271edb6fa7f672d4bde424eea2c61b746d13edef7","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20a552c9ef65c7031e4db49271edb6fa7f672d4bde424eea2c61b746d13edef7","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T14:03:14Z\n  Ended:       2026-10-07T14:20:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9974528-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T04:40:19Z\n  Ended:       2026-10-07T04:55:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2737280-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"125a20f06d40b81d0b44090ea3789c5fe4521773a1cc660485637fe5bfa1afbb","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"125a20f06d40b81d0b44090ea3789c5fe4521773a1cc660485637fe5bfa1afbb","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":14,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":40,"prior_last_seen_at":"2026-10-09T08:49:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-09T10:08:14Z\n  Ended:       2026-10-09T10:15:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4483328-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4483328-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-09T10:31:13Z\n  Ended:       2026-10-09T10:40:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4483328-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4483328-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-08T23:31:34Z\n  Ended:       2026-10-09T00:04:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4990592-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j4990592-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","occurred_at":"2026-10-09T08:49:50Z","signature_id":"2321790741ede5cfc4112f5540dcfb98866ee514350ef355bda4927b9c6a5c1d","pr_number":6944,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","occurred_at":"2026-10-08T22:02:25Z","signature_id":"9dd9aa3c94a6d8b3b046f06122de3e26069f01c693ef0bd9d5a924fe0d0972a0","pr_number":7364,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T17:20:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T18:38:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j1936896-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/services-j1936896\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of completed cluster create (install) operations on ci01-j1936896-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d53a2236830f49d3e673f190f6889ac45bcba408a0f6c004202adfc7dcecbe92","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"833cf2b5be700efed721d7f0826a89d849bf5421fcf520d4fb7e22e13dcd387a","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-09T17:15:54Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-12T17:06:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T19:03:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j1936896-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/services-j1936896\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j1936896-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"f0a5ca335c5452170de0c7d7ae55e4a8594a4284ebc53b77ca227c096a57881b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","signature_id":"df684fe7f4e6c9e63460594adb2966f6b38fe7c13181dc1da0d9f3bfdbb319bb","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"da6ef9b7d4b10235ac25ede586d99e51451945ec64f5760ee4de6b9d47ea200e","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed starting nodepool creation \"np-1\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\": PUT \u003curl\u003e","search_query":"failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-fh65w65xjrlv\": PUT https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T19:32:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e182923c240\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ls445v45mtp4\": PUT https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-ls445v45mtp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e182923c240\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ls445v45mtp4\": PUT https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-ls445v45mtp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:162]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e9eea24a6c0\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-fh65w65xjrlv\": PUT https://rp.j8091008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-fh65w65xjrlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:162]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e9eea24a6c0\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-fh65w65xjrlv\": PUT https://rp.j8091008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-fh65w65xjrlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"3fd8a931a3e5bf9db95183328d35b726f2c6c6c4e8ee953ac0421be7cfbc1384","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107817608098091008","occurred_at":"2026-10-07T12:57:33Z","signature_id":"6865a4e5afd5f302e897a1e9b8a117f3650268c89570c1a3a5cb4b887706e488","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-09T16:24:33Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-09T16:49:10.013Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-09T16:49:10.529Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-09T16:49:10.529Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-09T16:49:11.005Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations\ntime=2026-10-09T16:49:11.316Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false\ntime=2026-10-09T16:49:11.316Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-09T16:49:11.444Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-09T16:55:10.359Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-09T16:55:10.360Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-10-09T15:19:24.628Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-09T15:19:25.146Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-09T15:19:25.146Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-09T15:19:25.687Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-09T15:19:25.942Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-09T15:19:25.942Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-09T15:19:26.093Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-09T15:25:12.869Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-09T15:25:12.869Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108594476522934272","occurred_at":"2026-10-09T16:24:33Z","signature_id":"3eea31dd2ba80ae3c26eda9638a5c4961c637f75b7b51fe4771e43332561b111","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108567720399212544","occurred_at":"2026-10-09T14:38:14Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-12T10:35:56.708Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:57.601Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:35:57.906Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:35:57.907Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:58.041Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:40:59.966Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:40:59.967Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"","time=2026-10-12T10:30:06.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:07.671Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:08.275Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:10.285Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:35:10.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","pr_number":7439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-10-05"],"prior_runs_affected":18,"prior_last_seen_at":"2026-10-09T02:34:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":2}],"full_error_samples":["time=2026-10-09T03:02:25.127Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-09T03:02:27.026Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-09T03:02:29.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9100c383496433b8981e04307d4ff94d8b8595203688da3bb283a6fce4e49fea portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9299456-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9100c383496433b8981e04307d4ff94d8b8595203688da3bb283a6fce4e49fea\ntime=2026-10-09T03:32:24.844Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-10-08T14:39:49.369Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-08T14:39:51.193Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-08T14:39:53.050Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4570d5430c346685aef785faa645617cca8a23038712be321dc348c5ac428645 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1446656-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4570d5430c346685aef785faa645617cca8a23038712be321dc348c5ac428645\ntime=2026-10-08T15:09:49.053Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108385613169299456","occurred_at":"2026-10-09T02:34:36Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":7394,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","occurred_at":"2026-10-08T14:14:35Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-d5s257gp6qv6\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.6430868167202572,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-07T14:32:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x50103f9c9c0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-d5s257gp6qv6\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-d5s257gp6qv6\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x50103f9c9c0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-d5s257gp6qv6\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-d5s257gp6qv6\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c581ef3e4a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wcbt6xkttfqm\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-wcbt6xkttfqm\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c581ef3e4a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wcbt6xkttfqm\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-wcbt6xkttfqm\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107841500657225728","occurred_at":"2026-10-07T14:32:30Z","signature_id":"fec7daaa523868402b6b536ad9a8369cf346299cb7d8ea871ac465218dc8895b","pr_number":6954,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"420046f20ff960ccdda2a598e04cfcbca82502afdbf5b94e7a99272cdc27ca96","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8408b249004f1eb1294f56285dbfbec0fecc715e2e28bc3573e8af1426314384","failure_pattern":"ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])","search_query":"ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T21:02:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:142]: Timed out after 1200.007s.\nInsights Operator did not reach Disabled/NoToken state, the NoToken-to-healthy transition cannot be tested\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x27811265a7d0\u003e: \n    ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])\n    {\n        s: \"ClusterOperator \\\"insights\\\" not yet in Disabled/NoToken state (conditions: [])\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:142]: Timed out after 1200.007s.\nInsights Operator did not reach Disabled/NoToken state, the NoToken-to-healthy transition cannot be tested\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x27811265a7d0\u003e: \n    ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])\n    {\n        s: \"ClusterOperator \\\"insights\\\" not yet in Disabled/NoToken state (conditions: [])\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107939651057618944","occurred_at":"2026-10-07T21:02:31Z","signature_id":"748b7886a08fb3fd1cc9a8d22e3de74573c8fbb8f8d74801689482e69f7cff3c","pr_number":6954,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-09T21:37:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["77f90 sp=0x3ae32ce77ed0 pc=0x586c752\nnet/http.(*Transport).startDialConnForLocked.func1()\n\t/usr/lib/golang/src/net/http/transport.go:1629 +0x35 fp=0x3ae32ce77fe0 sp=0x3ae32ce77f90 pc=0x586c595\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32ce77fe8 sp=0x3ae32ce77fe0 pc=0x5540381\ncreated by net/http.(*Transport).startDialConnForLocked in goroutine 119\n\t/usr/lib/golang/src/net/http/transport.go:1628 +0x112\n\ngoroutine 167 gp=0x3ae32ce014a0 m=nil [semacquire]:\nruntime.gopark(0x3f00400063cb02?, 0x3ae32da24402?, 0xe0?, 0xc0?, 0x3ae32d696bb8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x3ae32d6969c0 sp=0x3ae32d6969a0 pc=0x553818e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x16f798a4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x3ae32d696a28 sp=0x3ae32d6969c0 pc=0x5513872\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x3ae32d696ac8 sp=0x3ae32d696a28 pc=0x54d6232\nruntime.gcAssistAlloc(0x3ae32ce014a0)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x3ae32d696b58 sp=0x3ae32d696ac8 pc=0x54dadd6\nruntime.deductAssistCredit(0x1?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x3ae32d696b78 sp=0x3ae32d696b58 pc=0x54cb6a6\nruntime.mallocgc(0x590, 0xf965760, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x3ae32d696ba8 sp=0x3ae32d696b78 pc=0x5534e67\nruntime.newobject(0x3ae32da2445d?)\n\t/usr/lib/golang/src/runtime/malloc.go:2157 +0x25 fp=0x3ae32d696bd0 sp=0x3ae32d696ba8 pc=0x54cb7a5\ncrypto/x509.parseCertificate({0x3ae32da4c000, 0x56d, 0x582})\n\t/usr/lib/golang/src/crypto/x509/parser.go:887 +0x45 fp=0x3ae32d697068 sp=0x3ae32d696bd0 pc=0x5729285\ncrypto/x509.ParseCertificate({0x3ae32da4c000?, 0x56d, 0x2ea83?})\n\t/usr/lib/golang/src/crypto/x509/parser.go:1077 +0x1d fp=0x3ae32d697090 sp=0x3ae32d697068 pc=0x572a69d\ncrypto/x509.(*CertPool).AppendCertsFromPEM(0x3ae32d4196e0, {0x3ae32da80000?, 0x100a51a0?, 0x10059470?})\n\t/usr/lib/golang/src/crypto/x509/cert_pool.go:231 +0x125 fp=0x3ae32d6971b0 sp=0x3ae32d697090 pc=0x571d2c5\ncrypto/x509.loadSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root_unix.go:72 +0x391 fp=0x3ae32d697298 sp=0x3ae32d6971b0 pc=0x572ccf1\ncrypto/x509.initSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root.go:43 +0x65 fp=0x3ae32d6972d8 sp=0x3ae32d697298 pc=0x572c785\nsync.(*Once).doSlow(0x54ca9ea?, 0x1f1a89e3ef160b9?)\n\t/usr/lib/golang/src/sync/once.go:78 +0xac fp=0x3ae32d697330 sp=0x3ae32d6972d8 pc=0x555198c\nsync.(*Once).Do(...)\n\t/usr/lib/golang/src/sync/once.go:69\ncrypto/x509.systemRootsPool()\n\t/usr/lib/golang/src/crypto/x509/root.go:32 +0x45 fp=0x3ae32d697380 sp=0x3ae32d697330 pc=0x572c645\ncrypto/x509.(*Certificate).Verify(0x3ae32d4ae008, {{0x3ae32d1d9be0, 0x19}, 0x3ae32d419620, 0x0, {0xc2aa7a68520d364e, 0xb6db3cc, 0x16f76d00}, {0x0, 0x0, ...}, ...})\n\t/usr/lib/golang/src/crypto/x509/verify.go:578 +0x18b fp=0x3ae32d697530 sp=0x3ae32d697380 pc=0x572f5cb\ncrypto/tls.(*Conn).verifyServerCertificate(0x3ae32d432008, {0x3ae32d49a5a0, 0x3, 0x4})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:1168 +0xb0e fp=0x3ae32d6978b8 sp=0x3ae32d697530 pc=0x5781dee\ncrypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:640 +0x29a fp=0x3ae32d697ab0 sp=0x3ae32d6978b8 pc=0x578629a\ncrypto/tls.(*clientHandshakeStateTLS13).handshake(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:141 +0x7cc fp=0x3ae32d697b70 sp=0x3ae32d697ab0 pc=0x578344c\ncrypto/tls.(*Conn).clientHandshake(0x3ae32d432008, {0x1018a380, 0x3ae32ca24690})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:339 +0x828 fp=0x3ae32d697d88 sp=0x3ae32d697b70 pc=0x577cb68\ncrypto/tls.(*Conn).clientHandshake-fm({0x1018a380?, 0x3ae32ca24690?})\n\t\u003cautogenerated\u003e:1 +0x2a fp=0x3ae32d697db0 sp=0x3ae32d697d88 pc=0x57bcaca\ncrypto/tls.(*Conn).handshakeContext(0x3ae32d432008, {0x1018a380, 0x3ae32c9f4460})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1562 +0x329 fp=0x3ae32d697f70 sp=0x3ae32d697db0 pc=0x5773829\ncrypto/tls.(*Conn).HandshakeContext(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:1516\nnet/http.(*persistConn).addTLS.func2()\n\t/usr/lib/golang/src/net/http/transport.go:1736 +0x6e fp=0x3ae32d697fe0 sp=0x3ae32d697f70 pc=0x586d3ce\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32d697fe8 sp=0x3ae32d697fe0 pc=0x5540381\ncreated by net/http.(*persistConn).addTLS in goroutine 120\n\t/usr/lib/golang/src/net/http/transport.go:1732 +0x309\n\nrax    0x0\nrbx    0x7fe37a988640\nrcx    0x7fe3c265d94c\nrdx    0x6\nrdi    0xa6f0\nrsi    0xa6f4\nrbp    0xa6f4\nrsp    0x7fe37a987790\nr8     0x7fe37a987860\nr9     0x7fe3c27834e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7fe37a987b60\nr14    0x3ae32bf97860\nr15    0xffffffffffffffff\nrip    0x7fe3c265d94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n77f90 sp=0x3ae32ce77ed0 pc=0x586c752\nnet/http.(*Transport).startDialConnForLocked.func1()\n\t/usr/lib/golang/src/net/http/transport.go:1629 +0x35 fp=0x3ae32ce77fe0 sp=0x3ae32ce77f90 pc=0x586c595\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32ce77fe8 sp=0x3ae32ce77fe0 pc=0x5540381\ncreated by net/http.(*Transport).startDialConnForLocked in goroutine 119\n\t/usr/lib/golang/src/net/http/transport.go:1628 +0x112\n\ngoroutine 167 gp=0x3ae32ce014a0 m=nil [semacquire]:\nruntime.gopark(0x3f00400063cb02?, 0x3ae32da24402?, 0xe0?, 0xc0?, 0x3ae32d696bb8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x3ae32d6969c0 sp=0x3ae32d6969a0 pc=0x553818e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x16f798a4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x3ae32d696a28 sp=0x3ae32d6969c0 pc=0x5513872\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x3ae32d696ac8 sp=0x3ae32d696a28 pc=0x54d6232\nruntime.gcAssistAlloc(0x3ae32ce014a0)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x3ae32d696b58 sp=0x3ae32d696ac8 pc=0x54dadd6\nruntime.deductAssistCredit(0x1?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x3ae32d696b78 sp=0x3ae32d696b58 pc=0x54cb6a6\nruntime.mallocgc(0x590, 0xf965760, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x3ae32d696ba8 sp=0x3ae32d696b78 pc=0x5534e67\nruntime.newobject(0x3ae32da2445d?)\n\t/usr/lib/golang/src/runtime/malloc.go:2157 +0x25 fp=0x3ae32d696bd0 sp=0x3ae32d696ba8 pc=0x54cb7a5\ncrypto/x509.parseCertificate({0x3ae32da4c000, 0x56d, 0x582})\n\t/usr/lib/golang/src/crypto/x509/parser.go:887 +0x45 fp=0x3ae32d697068 sp=0x3ae32d696bd0 pc=0x5729285\ncrypto/x509.ParseCertificate({0x3ae32da4c000?, 0x56d, 0x2ea83?})\n\t/usr/lib/golang/src/crypto/x509/parser.go:1077 +0x1d fp=0x3ae32d697090 sp=0x3ae32d697068 pc=0x572a69d\ncrypto/x509.(*CertPool).AppendCertsFromPEM(0x3ae32d4196e0, {0x3ae32da80000?, 0x100a51a0?, 0x10059470?})\n\t/usr/lib/golang/src/crypto/x509/cert_pool.go:231 +0x125 fp=0x3ae32d6971b0 sp=0x3ae32d697090 pc=0x571d2c5\ncrypto/x509.loadSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root_unix.go:72 +0x391 fp=0x3ae32d697298 sp=0x3ae32d6971b0 pc=0x572ccf1\ncrypto/x509.initSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root.go:43 +0x65 fp=0x3ae32d6972d8 sp=0x3ae32d697298 pc=0x572c785\nsync.(*Once).doSlow(0x54ca9ea?, 0x1f1a89e3ef160b9?)\n\t/usr/lib/golang/src/sync/once.go:78 +0xac fp=0x3ae32d697330 sp=0x3ae32d6972d8 pc=0x555198c\nsync.(*Once).Do(...)\n\t/usr/lib/golang/src/sync/once.go:69\ncrypto/x509.systemRootsPool()\n\t/usr/lib/golang/src/crypto/x509/root.go:32 +0x45 fp=0x3ae32d697380 sp=0x3ae32d697330 pc=0x572c645\ncrypto/x509.(*Certificate).Verify(0x3ae32d4ae008, {{0x3ae32d1d9be0, 0x19}, 0x3ae32d419620, 0x0, {0xc2aa7a68520d364e, 0xb6db3cc, 0x16f76d00}, {0x0, 0x0, ...}, ...})\n\t/usr/lib/golang/src/crypto/x509/verify.go:578 +0x18b fp=0x3ae32d697530 sp=0x3ae32d697380 pc=0x572f5cb\ncrypto/tls.(*Conn).verifyServerCertificate(0x3ae32d432008, {0x3ae32d49a5a0, 0x3, 0x4})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:1168 +0xb0e fp=0x3ae32d6978b8 sp=0x3ae32d697530 pc=0x5781dee\ncrypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:640 +0x29a fp=0x3ae32d697ab0 sp=0x3ae32d6978b8 pc=0x578629a\ncrypto/tls.(*clientHandshakeStateTLS13).handshake(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:141 +0x7cc fp=0x3ae32d697b70 sp=0x3ae32d697ab0 pc=0x578344c\ncrypto/tls.(*Conn).clientHandshake(0x3ae32d432008, {0x1018a380, 0x3ae32ca24690})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:339 +0x828 fp=0x3ae32d697d88 sp=0x3ae32d697b70 pc=0x577cb68\ncrypto/tls.(*Conn).clientHandshake-fm({0x1018a380?, 0x3ae32ca24690?})\n\t\u003cautogenerated\u003e:1 +0x2a fp=0x3ae32d697db0 sp=0x3ae32d697d88 pc=0x57bcaca\ncrypto/tls.(*Conn).handshakeContext(0x3ae32d432008, {0x1018a380, 0x3ae32c9f4460})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1562 +0x329 fp=0x3ae32d697f70 sp=0x3ae32d697db0 pc=0x5773829\ncrypto/tls.(*Conn).HandshakeContext(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:1516\nnet/http.(*persistConn).addTLS.func2()\n\t/usr/lib/golang/src/net/http/transport.go:1736 +0x6e fp=0x3ae32d697fe0 sp=0x3ae32d697f70 pc=0x586d3ce\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32d697fe8 sp=0x3ae32d697fe0 pc=0x5540381\ncreated by net/http.(*persistConn).addTLS in goroutine 120\n\t/usr/lib/golang/src/net/http/transport.go:1732 +0x309\n\nrax    0x0\nrbx    0x7fe37a988640\nrcx    0x7fe3c265d94c\nrdx    0x6\nrdi    0xa6f0\nrsi    0xa6f4\nrbp    0xa6f4\nrsp    0x7fe37a987790\nr8     0x7fe37a987860\nr9     0x7fe3c27834e0\nr10    0x8\nr11    0x246\nr12    0x6\nr13    0x7fe37a987b60\nr14    0x3ae32bf97860\nr15    0xffffffffffffffff\nrip    0x7fe3c265d94c\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","occurred_at":"2026-10-09T21:37:50Z","signature_id":"ea5426c1ae50dc34a6a959cb8f4ddaf157e2c2b692d50dd5d9b193626990f1eb","pr_number":7446,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22fbdb35a870cfc5f2c9e8846465e0b7f85e96d6e430537b6f1feed38d45976b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code KeyVaultAccessForbidden; detail message Unable to access key vault resource '\u003curl\u003e' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n      \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n    \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n            correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n            client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n      \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n    \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n            correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n            client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"a902cb53b68f347d3389b746e7a2281dfe680f58c4b7f49f68ca93a281c00759","pr_number":7470,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64c6e11cbb74bffe980698f0ac18939b0cc44bb682a44aa8b5b4f43aee00cd17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n      \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n            correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n            client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n      \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n            correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n            client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","signature_id":"93e4587d665844128088f4ce3a0a7ac0670bc2eccb4898d241b67e1ee37e48d9","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27aef5ac99f567baefaad85029e0d3da9b4dbe40c6bbe0be985920a793730d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n      \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n            correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n            client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n      \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n            correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n            client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"59da2ca63ee261235a31615de260be397d214b0f4f694b160b2ac972a17ed31e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b89623a31d2776290135b67a1c3e8137de86d30ee19f034e5531fb48ad00d00f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n      \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n            correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n            client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n    failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n      \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n            correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n            client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"01fffc7b6758239c11fdf9bc2bb6322f5a22ddef4368d43104f752d3e34ce9e2","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-30T16:03:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n      \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n            correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n            client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n      \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n            correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n            client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"317dedee167ea472d7b8eb2f6ca44fba55d08af18e00d84e205137d4efe864b6","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9da4ad2c21fc0e265207db4c5abcfdd0631935b407c3edcd41248ee669719f25","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n      \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n            correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n            client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n      \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n            correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n            client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"6ca522238814ece9529f702529a6bcb452636d575084a8828e41559f858abbbd","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a69ffbb8a895af0747c6f4a476af227a32150aff47581bb3fd7f521ea56ce13e","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-11T15:50:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n      \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n            correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n            client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2952d6db0d20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-ksddfkp5xbq5\" to finish creating: GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"name\": \"6fd0636d-925f-45c9-a7fc-b6aa390b633a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T16:48:04.203625548Z\",\n      \"endTime\": \"2026-10-11T17:17:06.363062447Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkw9f-sxlv4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkw9f-sxlv4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ksddfkp5xbq5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkw9f-sxlv4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8626176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fd0636d-925f-45c9-a7fc-b6aa390b633a\n            correlation ID: 9211dc6b-1604-468e-b90e-8381473929df\n            client request ID: 967f10bc-a33e-4e81-877c-ed8ff3004e75\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109310669768626176","occurred_at":"2026-10-11T15:50:27Z","signature_id":"929e7bd719fc59b94777a6ea0a168be5415092782f3e6702397acd6adaa7c232","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"03ddc6ab96e7bba4116d23faa9db196492be0ac8bee0a6170184bf1e8ecbc394","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-10T05:02:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24a46d574920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlxq4qgj7xpp\" to finish creating: GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\",\n      \"name\": \"309223c5-c91d-4270-b774-50dec66e2570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T06:15:46.054292016Z\",\n      \"endTime\": \"2026-10-10T06:44:53.542490042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-drwbc-mbw6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-drwbc-mbw6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n            correlation ID: 57417513-291c-4d90-a0bc-a093da0940a4\n            client request ID: 3e97b4e2-ed3b-4f11-be4f-69ec5d075229\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24a46d574920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-mlxq4qgj7xpp\" to finish creating: GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\",\n      \"name\": \"309223c5-c91d-4270-b774-50dec66e2570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-10T06:15:46.054292016Z\",\n      \"endTime\": \"2026-10-10T06:44:53.542490042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-drwbc-mbw6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-drwbc-mbw6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mlxq4qgj7xpp--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-drwbc-mbw6z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0639104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/309223c5-c91d-4270-b774-50dec66e2570\n            correlation ID: 57417513-291c-4d90-a0bc-a093da0940a4\n            client request ID: 3e97b4e2-ed3b-4f11-be4f-69ec5d075229\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108785286350639104","occurred_at":"2026-10-10T05:02:46Z","signature_id":"5ed4b18dce6b054ef58ec3f72cbaa45d685da5b7aa78958846e995ebb0a58680","pr_number":7227,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2c785268dead82f5391d67011a5ed778899c269360f3bc0b5673c6c7eb09c779","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; hypershift NodePool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; hypershift NodePool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T16:48:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0b67484b00\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-b6bhhgvdr2ld\" to finish updating: GET https://rp.j0967424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"name\": \"9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:15:51.4879876Z\",\n      \"endTime\": \"2026-10-07T18:34:57.661146093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool status replicas is 2, want 3; hypershift NodePool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0b67484b00\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-b6bhhgvdr2ld\" to finish updating: GET https://rp.j0967424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"name\": \"9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:15:51.4879876Z\",\n      \"endTime\": \"2026-10-07T18:34:57.661146093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool status replicas is 2, want 3; hypershift NodePool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7322/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875761300967424","occurred_at":"2026-10-07T16:48:38Z","signature_id":"3eddb723e07465e8928579f25de472d49a61e72fa22ed9364fe0258841f58a49","pr_number":7322,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"531a2084e8686f9a42755ac056f2dd31faa67fc9e5c5979d05857ee0dfb1b975","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd4464611460\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n            correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd4464611460\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n            correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"01033c5c618998eef4c756c29b116f1c2e1c20ad91b1f1d16200cebb174aafe7","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d1a2d3dadcb1c305fd461d5e9526c1e1f98093bc5d6545af22d4aa0cf7e7d759","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T02:42:16Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-08T03:34:00.150Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-08T03:34:00.987Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-08T03:34:02.023Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9563392-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220\ntime=2026-10-08T03:36:02.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.Resources/deployments/058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220/operationStatuses/08584101764439358601\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mgmt-agent' was not found.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"NotFound\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025152309563392","occurred_at":"2026-10-08T02:42:16Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RequestTimeout; detail message Database account creation failed. Operation Id: \u003cuuid\u003e, Error : Message: {\"Errors\":[\"The requested operation exceeded maximum alloted time. Learn more: \u003curl\u003e; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-10-12T23:56:30.391Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-12T23:56:32.064Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-12T23:56:32.955Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3444352%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3\ntime=2026-10-13T00:03:04.471Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3444352/providers/Microsoft.Resources/deployments/2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3/operationStatuses/08584097574930206657\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3444352/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3444352/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j3444352\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 42664daf-4e8e-4916-9d75-23b7169468da, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 26eaccdb-de17-4cbf-afc1-0c865cfbb1a2, Request URI: /apps/de70b74f-b005-4de6-b2c3-b39046bf9084/services/74353989-92b9-4a77-a49a-b47600be8b14/partitions/a40f5539-0cc6-43c1-b5bc-fdb8dcb67485/replicas/134362021132042649s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-13T00:02:20.9415661Z, RequestEndTime: 2026-10-13T00:02:26.9109610Z,  Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:28.0336227Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.273,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21877192.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1501,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:38.0493247Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.521,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21840180.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0747,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:48.0505987Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21844420.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1332,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:58.0664654Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.975,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21830384.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1746,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:02:08.0821422Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21833552.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0846,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:02:18.0977872Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":2.281,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21805124.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1832,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":51}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-13T00:02:20.9415661Z; ResponseTime: 2026-10-13T00:02:26.9109610Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be424.documents.azure.com:11306/apps/de70b74f-b005-4de6-b2c3-b39046bf9084/services/74353989-92b9-4a77-a49a-b47600be8b14/partitions/a40f5539-0cc6-43c1-b5bc-fdb8dcb67485/replicas/134362021132042649s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 5975.386, ActivityId: 26eaccdb-de17-4cbf-afc1-0c865cfbb1a2, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11306 | status: Connected | lkt: 10/13/2026 12:02:26 AM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415661Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0013},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415674Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.006},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415734Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0628},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9416362Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 5976.0734},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:26.9177096Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.1891},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:26.9178987Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:07.6413599Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:07.6413599Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:12.9085149Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109790061913444352","occurred_at":"2026-10-12T23:35:23Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7483,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64bb2607db9a31af645f91fcf6ffbdb70d8bca63638eea03e861ce2387c49227","failure_pattern":"ERROR CODE: DeploymentNotFound; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-09T21:41:16Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-09T22:05:07.216Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-09T22:06:09.182Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-09T22:05:07.225651655Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-09T22:05:08.960792766Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-09T22:05:09.292773856Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-09T22:06:09.178268762Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":{\\\"errorCode\\\":\\\"DeploymentNotFound\\\",\\\"statusCode\\\":404,\\\"errorMessage\\\":\\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7960192-mgmt-2/deployments/aks-cluster-create/operations\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE: DeploymentNotFound\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'aks-cluster-create' could not be found.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"}}\\n{\\\"time\\\":\\\"2026-10-09T22:06:09.178359704Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"submitting deployment aks-cluster-create: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7960192-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 400: 400 Bad Request\\\\nERROR CODE: InvalidTemplateDeployment\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'aks-cluster-create' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.ContainerService/managedClusters (2026-04-02-preview)' reported preflight validation errors. Tracking id is 'af747446-bfb7-43e9-a875-3f8953ec8e67'. See inner errors for details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ListResourceSKUsFailed\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"Preflight validation check for resource(s) for container service ci01-j7960192-mgmt-2 in resource group hcp-underlay-ci01-j7960192-mgmt-2 failed. Message: List resource SKUs in subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX failed. Details: \\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"1m1.952725139s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108674180797960192","occurred_at":"2026-10-09T21:41:16Z","signature_id":"abe4c5c2b231f43f69eb7923fecd7d0c54d26cc4bcd5d781e5832765ceb537cc","pr_number":7249,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"03191c7ee6ef10f05815abde7f8b6f6bad2b4b3b1acdea0c834f5ddce823bc91","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T16:49:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3974085580d8\u003e: \n    resource group \"external-auth-cluster-jkpp8mdvzxsh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jkpp8mdvzxsh\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"name\": \"7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:50.350274496Z\",\n      \"endTime\": \"2026-10-07T18:24:57.82988818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3adt66dtiffarpvt6smilc2f19037 still exists (deletion dispatched at 2026-10-07T18:00:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3974085580d8\u003e: \n    resource group \"external-auth-cluster-jkpp8mdvzxsh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jkpp8mdvzxsh\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"name\": \"7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:50.350274496Z\",\n      \"endTime\": \"2026-10-07T18:24:57.82988818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3adt66dtiffarpvt6smilc2f19037 still exists (deletion dispatched at 2026-10-07T18:00:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"ad65d918080739de6f2f3b2f425a73e3b32c8cc57bfe3363cce8fe1a1dc681fd","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-08T11:12:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1ae233ff2078\u003e: \n    resource group \"v20260901preview-hqg6wzs5tgtz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-hqg6wzs5tgtz\" to finish deleting: GET https://rp.j2977792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\",\n      \"name\": \"b7f165f5-05d4-46b6-9b64-0134316e8951\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T12:28:19.710418085Z\",\n      \"endTime\": \"2026-10-08T12:52:26.28519628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tajia53q41l8grpaq7k8eu3qvn81ojn still exists (deletion dispatched at 2026-10-08T12:28:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1ae233ff2078\u003e: \n    resource group \"v20260901preview-hqg6wzs5tgtz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-hqg6wzs5tgtz\" to finish deleting: GET https://rp.j2977792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\",\n      \"name\": \"b7f165f5-05d4-46b6-9b64-0134316e8951\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T12:28:19.710418085Z\",\n      \"endTime\": \"2026-10-08T12:52:26.28519628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tajia53q41l8grpaq7k8eu3qvn81ojn still exists (deletion dispatched at 2026-10-08T12:28:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","signature_id":"2fd4ac18224462c0996aff2443f5feed4b4cd3d6b83054edc878630f518b2ab6","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T16:47:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3f397df201c8\u003e: \n    resource group \"clusternp128-89h59bcbsmnq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-89h59bcbsmnq\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"name\": \"84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:09.863415259Z\",\n      \"endTime\": \"2026-10-07T18:24:11.307794702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3chch61ipf94injnrljj1hk0caf92 still exists (deletion dispatched at 2026-10-07T18:00:10Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3f397df201c8\u003e: \n    resource group \"clusternp128-89h59bcbsmnq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-89h59bcbsmnq\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"name\": \"84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:09.863415259Z\",\n      \"endTime\": \"2026-10-07T18:24:11.307794702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3chch61ipf94injnrljj1hk0caf92 still exists (deletion dispatched at 2026-10-07T18:00:10Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"cc042585e1ff82e3d4f824868e1321ec45e4977ce868ac923076ba255c0d42a1","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db15661ea057fcdb86696aa934e0ebb9321ec309068c024420faf52e725ffb06","failure_pattern":"ERROR: Please run 'az login' to setup account.","search_query":"ERROR: Please run 'az login' to setup account.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T08:31:41Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-07T09:04:20.332Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-10-07T09:04:22.138Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nERROR: Please run 'az login' to setup account.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750698572582912","occurred_at":"2026-10-07T08:31:41Z","signature_id":"21658b9b3d6cf9bf31715ba32c333953291d0d0fe4a6e4e41252f24603e05dff","pr_number":7398,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1ae9bcc864e0b95b17cfb70f07fc859ef6b54ed4d13d4886751e4ae7fe6a17c0","failure_pattern":"Internal error occurred: failed calling webhook \"webhook","search_query":"Internal error occurred: failed calling webhook \"webhook","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T08:29:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:337]: Timed out after 120.000s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x3dfc585d0640\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:337]: Timed out after 120.000s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x3dfc585d0640\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750168211230720","occurred_at":"2026-10-07T08:29:34Z","signature_id":"aa5a79bb72cb1d70468a298f56bdb83213121c034823482ee690c746113359f8","pr_number":7370,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T03:35:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:128]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0x3fa9a7564d98\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3fa9a6cff3e0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6cff3c0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a7398db0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0x3fa9a716a690\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3fa9a6ee5c60\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6ee5c40\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a786a420\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0x3fa9a7c7d0e0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:128]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0x3fa9a7564d98\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3fa9a6cff3e0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6cff3c0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a7398db0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0x3fa9a716a690\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3fa9a6ee5c60\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6ee5c40\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a786a420\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0x3fa9a7c7d0e0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"9f080afeacc6660909212d05408fd022c6a74fba0cb0a9759c7ec00f6ad7192d","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-08T02:42:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0x3e609f8540a8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3e609fd10180\u003e{\n                msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                err: \u003c*fmt.wrapError | 0x3e609fd10160\u003e{\n                    msg: \"failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                    err: \u003c*fmt.wrapError | 0x3e609fd10020\u003e{\n                        msg: \"failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                        err: \u003c*errors.StatusError | 0x3e609eee2c80\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                                Reason: \"InternalError\",\n                                Details: {\n                                    Name: \"fips-check-9c9j4\",\n                                    Group: \"\",\n                                    Kind: \"pods\",\n                                    UID: \"\",\n                                    Causes: [\n                                        {Type: ..., Message: ..., Field: ...},\n                                    ],\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 500,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0x3e609f8540a8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3e609fd10180\u003e{\n                msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                err: \u003c*fmt.wrapError | 0x3e609fd10160\u003e{\n                    msg: \"failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                    err: \u003c*fmt.wrapError | 0x3e609fd10020\u003e{\n                        msg: \"failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                        err: \u003c*errors.StatusError | 0x3e609eee2c80\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n                                Reason: \"InternalError\",\n                                Details: {\n                                    Name: \"fips-check-9c9j4\",\n                                    Group: \"\",\n                                    Kind: \"pods\",\n                                    UID: \"\",\n                                    Causes: [\n                                        {Type: ..., Message: ..., Field: ...},\n                                    ],\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 500,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025105396273152","occurred_at":"2026-10-08T02:42:04Z","signature_id":"b6ae0ddb16ff4c94221800e11be0c27c614a44b85087bb2bbb64f1a59d3dee88","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1b8edd8bf3beb856400cfdf84db0af55a47389abcf6ab46ec6999548fb8b8f61","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-tkqcj)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T19:32:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xf6f46a980d8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-tkqcj)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xf6f45174120\u003e{\n                msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                err: \u003c*fmt.wrapError | 0xf6f45174100\u003e{\n                    msg: \"failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                    err: \u003c*fmt.wrapError | 0xf6f451742a0\u003e{\n                        msg: \"failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                        err: \u003c*errors.StatusError | 0xf6f4458f7c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                                Reason: \"InternalError\",\n                                Details: {\n                                    Name: \"fips-check-tkqcj\",\n                                    Group: \"\",\n                                    Kind: \"pods\",\n                                    UID: \"\",\n                                    Causes: [\n                                        {Type: ..., Message: ..., Field: ...},\n                                    ],\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 500,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xf6f46a980d8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-tkqcj)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xf6f45174120\u003e{\n                msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                err: \u003c*fmt.wrapError | 0xf6f45174100\u003e{\n                    msg: \"failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                    err: \u003c*fmt.wrapError | 0xf6f451742a0\u003e{\n                        msg: \"failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                        err: \u003c*errors.StatusError | 0xf6f4458f7c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                                Reason: \"InternalError\",\n                                Details: {\n                                    Name: \"fips-check-tkqcj\",\n                                    Group: \"\",\n                                    Kind: \"pods\",\n                                    UID: \"\",\n                                    Causes: [\n                                        {Type: ..., Message: ..., Field: ...},\n                                    ],\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 500,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"b2817992361bcd87227b455397537c7fa2c5d587900b532a6d346247a6380362","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2ea683689d213a80fd9693db9b372fae798477a56e7e530874ac1f5a4eddf2fd","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T12:57:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d103f80a620\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        err: \u003c*errors.errorString | 0x1d103f3186b0\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d103f80a620\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        err: \u003c*errors.errorString | 0x1d103f3186b0\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107817608098091008","occurred_at":"2026-10-07T12:57:33Z","signature_id":"7a2c94fb45e7db0296dcd9f041aa0c8c5dcbc38fae73401939da42d841d4b069","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0a9bcf49d179171c63dccafd8dbb4e653af27553fa78df236def98914f6ada90","failure_pattern":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df811d0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df140c0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n                    msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019e2caff0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n                        Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df811d0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019df140c0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n                    msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0xb019e2caff0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n                        Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","signature_id":"a87f271596a6f4e182a266365db74092bd67d3130b10c9e295b2604bd480ee54","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":15,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":46,"prior_last_seen_at":"2026-10-02T20:20:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11d596138740\u003e{\n                msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n                    s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                },\n            },\n            \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n                msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n                    s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11d596138740\u003e{\n                msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n                    s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n                },\n            },\n            \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n                msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n                    s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e72a8b419adb6b53b3d96517db9d967450746716b4acc806c6d35e614fc4cc81","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-08T07:57:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x1409e4db1080\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1409e4324dc0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x1409e4f4c060\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x1409e4f4c040\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0x1409e4db1080\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1409e4324dc0\u003e{\n                msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x1409e4f4c060\u003e{\n                    msg: \"DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                    err: \u003c*fmt.wrapError | 0x1409e4f4c040\u003e{\n                        msg: \"DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n                        err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","occurred_at":"2026-10-08T07:57:36Z","signature_id":"99e7ae876a6bc4905e749326067bbde5a6d33b2d8c46d8ed6fbac42e335e17f3","pr_number":6775,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9626368 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9626368 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T16:24:31Z\n  Ended:       2026-10-12T16:44:33Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-09T07:01:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-09T08:14:32Z\n  Ended:       2026-10-09T08:32:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0006144-svc\", component=\"fleet\", name=\"nodepoolcontroller\", region=\"westus3\", severity=\"warning\"\n  Description: Fleet controller workqueue nodepoolcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","signature_id":"b1312384d9b2a53ea723368157d1cd9705e0e9f23816fc33749b5b53bb2fe8fb","pr_number":7437,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[3,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":24,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:20:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:20:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:20:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-09T15:20:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","signature_id":"c28dae8ce3537c2027112a365ecbd5dce6df30cf8f95741d318a215b7db8828b","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-07T22:28:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 9 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:27:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:27:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-07T23:34:27Z\n  Ended:       2026-10-07T23:43:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta829bp1fhgrfupabf34hjhbao8sjia\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta829bp1fhgrfupabf34hjhbao8sjia/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-07T23:35:31Z\n  Ended:       2026-10-07T23:45:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta82bca3t1j71tbsbphnabaj9385rs2\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta82bca3t1j71tbsbphnabaj9385rs2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-07T23:35:32Z\n  Ended:       2026-10-07T23:54:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-07T23:59:27Z\n  Ended:       2026-10-08T00:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8e00ategjs6jdt7dm9iouhjbvp29q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8e00ategjs6jdt7dm9iouhjbvp29q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-08T00:00:27Z\n  Ended:       2026-10-08T00:10:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8erefr60nd4djngqkit7pco8mj14u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8erefr60nd4djngqkit7pco8mj14u/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-08T00:00:27Z\n  Ended:       2026-10-08T00:07:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8f7d8ssfok4nfel8v6oomstor9osq\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8f7d8ssfok4nfel8v6oomstor9osq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-08T00:02:28Z\n  Ended:       2026-10-08T00:14:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8fb8j9edjipdiide2u4g57sv913hs\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8fb8j9edjipdiide2u4g57sv913hs/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107961335642853376","occurred_at":"2026-10-07T22:28:41Z","signature_id":"d5d12fa7c02227280374e42810634b4c5d7062a37a2bf81274486c206da8d33b","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-07T22:28:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:37:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-07T23:45:38Z\n  Ended:       2026-10-07T23:54:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T00:09:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107961335642853376","occurred_at":"2026-10-07T22:28:41Z","signature_id":"ae24b3bd209fd3bdae001beee3f0cc45e8c82e783e565d4b2501a33d06dca5b5","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":89,"prior_last_seen_at":"2026-10-05T01:04:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-12T16:40:12Z\n  Ended:       2026-10-12T16:50:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-56bb7dd747-h9s6h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-56bb7dd747-h9s6h (namespace aro-hcp) on cluster ci01-j9626368-svc has averaged 134.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-12T16:40:12Z\n  Ended:       2026-10-12T16:54:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-c6ff9b6f4-b8fh5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-c6ff9b6f4-b8fh5 (namespace kube-applier) on cluster ci01-j9626368-mgmt-2 has averaged 150.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-12T16:40:12Z\n  Ended:       2026-10-12T16:56:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-65fd757c56-d7zhs\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-65fd757c56-d7zhs (namespace kube-applier) on cluster ci01-j9626368-mgmt-1 has averaged 164.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"2ac18c0f28c3db988430e8cd508afbb6dc2ac259140c6a1cba13d912134a5113","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":37,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":109,"prior_last_seen_at":"2026-10-07T12:59:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:26:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9974528-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jrr5z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-jrr5z (namespace arobit) on cluster ci01-j9974528-svc has averaged 124.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"8a69335c6b425dfcfab867320cdd5348fb00e994d8030c8247190937dcfb5c68","pr_number":7407,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[1,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":15,"prior_last_seen_at":"2026-10-08T12:00:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3dc0df3e260\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3dc0df3e260\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"65b7540a6ca0b846f6cc9d88ae9a4e6476ca721e437f95cf3292bc42c3aa02d4","pr_number":7389,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5","failure_pattern":"console should return a successful response or redirect, got 503","search_query":"console should return a successful response or redirect, got 503","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-08T13:35:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:312]: Timed out after 2700.006s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:310 with:\nconsole should return a successful response or redirect, got 503\nExpected\n    \u003cint\u003e: 503\nto be \u003c\n    \u003cint\u003e: 400\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:312]: Timed out after 2700.006s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:310 with:\nconsole should return a successful response or redirect, got 503\nExpected\n    \u003cint\u003e: 503\nto be \u003c\n    \u003cint\u003e: 400"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108189567159898112","occurred_at":"2026-10-08T13:35:35Z","signature_id":"7f9cdf151f9d6c520b9b53bcf22e09916fb547d1ef82e84d3970472cca185ca2","pr_number":6807,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T16:13:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x17a8dd744af0?, {0x7f91c02f24c0?, 0x17a8dd261560?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1075 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x17a8dcfdf200, {0x7f91c02f24c0, 0x17a8dd261560}, {0xf9f89cb, 0x15}, {0x17a8dc60e129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:508 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func34.2({0x7f91c02f24c0, 0x17a8dd261560})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:68 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x101a3e60?, 0x17a8dd261560?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 134\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x17a8dd744af0?, {0x7f91c02f24c0?, 0x17a8dd261560?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1075 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x17a8dcfdf200, {0x7f91c02f24c0, 0x17a8dd261560}, {0xf9f89cb, 0x15}, {0x17a8dc60e129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:508 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func34.2({0x7f91c02f24c0, 0x17a8dd261560})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:68 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x101a3e60?, 0x17a8dd261560?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 134\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"c584d03b77f63107c8174575f5f10a9efc54b55c500b0c9908081f3f3d257f5a","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8ee2dc1e29d2efb4e37488a71b222c2bb43f2cba83c52cdf5162c1fd0f69c529","failure_pattern":"error for disabled boot diagnostics should contain 409 status code","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:456]: error for disabled boot diagnostics should contain 409 status code","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-09T01:49:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:456]: error for disabled boot diagnostics should contain 409 status code\nExpected\n    \u003cstring\u003e: failed to send request: Get \"https://admin.j1851392.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-mx7j6swwr25v/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster-bootdiag/serialconsole?vmName=d3i6t7c9q4h0k8j-worker-wrlpp-npg85\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nto contain substring\n    \u003cstring\u003e: 409\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:456]: error for disabled boot diagnostics should contain 409 status code\nExpected\n    \u003cstring\u003e: failed to send request: Get \"https://admin.j1851392.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-mx7j6swwr25v/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster-bootdiag/serialconsole?vmName=d3i6t7c9q4h0k8j-worker-wrlpp-npg85\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nto contain substring\n    \u003cstring\u003e: 409"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","signature_id":"9b3b2f939bd60c04510443c5b5a36de01dbb0d75878b8f7c5c390016031c92b1","pr_number":7437,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8d3193ed0a7e2c3492a5c226c9a0bdd4693921b8e35909048c2ef7aed3b7502d","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j4452224-mgmt-1\",\"clusterName\":\"ci01-j445222","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T14:32:31.068088679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T14:32:32.982339672Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T14:32:33.61885433Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T13:42:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-08T14:32:31.059Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T15:02:30.832Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T14:32:31.068088679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T14:32:32.982339672Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T14:32:33.61885433Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","occurred_at":"2026-10-08T13:42:51Z","signature_id":"454655692b823febcffc54365e504c69961bb8ad6b414fb6c4922c06be0ce0d9","pr_number":7427,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"647a5882391a49335381fff59c5e8cedbeb838f7f6bedacd6500d31f262d8c2e","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5137536-mgmt-2\",\"clusterName\":\"ci01-j513753","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-09T14:05:11.922891129Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.313036396Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.653688404Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-09T13:37:50Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-09T14:05:11.912Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-09T14:35:11.381Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-09T14:05:11.922891129Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.313036396Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.653688404Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108552522435137536","occurred_at":"2026-10-09T13:37:50Z","signature_id":"fa42174f7ac0fcceb8b39aba75fd064da702cc202084145000ae5032ccdb2c75","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3bc6a7df03bfd8951b086726c7bf177263d62d8ccd5624cb33a8a5e6706353f6","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T18:41:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x4aa5231ef20\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\n    let _endTime = datetime(2026-10-07T20:13:47.1720653Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-gg7xds6svpzk'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T20:13:54.1282773Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\", ActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", SubActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityStack:\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x4aa53d27068\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x4aa5231ef00\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x4aa5231eec0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x4aa52207ce0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x4aa52207ca0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x4aa52820300\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x4aa5231ef20\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\n    let _endTime = datetime(2026-10-07T20:13:47.1720653Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-gg7xds6svpzk'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T20:13:54.1282773Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\", ActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", SubActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityStack:\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x4aa53d27068\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x4aa5231ef00\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x4aa5231eec0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x4aa52207ce0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x4aa52207ca0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x4aa52820300\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"7d412545a42210e54d12fe8199b4fe8b2c91f6c65dc9ad7a4d53e143213f22db","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c7241dd3402b660f1519d58411034f82666a2c1c7f982d027aba3c2a37a8cdb1","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T21:02:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x339d2c183220\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\n    let _endTime = datetime(2026-10-07T22:32:32.2954057Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T22:32:43.6043688Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\", ActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", SubActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x339d2b3c97a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x339d2c183200\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x339d2c1831e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x339d2c1830c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x339d2c183080\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x339d2ae3d200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x339d2c183220\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\n    let _endTime = datetime(2026-10-07T22:32:32.2954057Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T22:32:43.6043688Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\", ActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", SubActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x339d2b3c97a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x339d2c183200\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x339d2c1831e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x339d2c1830c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x339d2c183080\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x339d2ae3d200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107939651057618944","occurred_at":"2026-10-07T21:02:31Z","signature_id":"707d524bc3708f6d8c9654e45c0eaa4ed86159ce6bbac3cc9b6a73ce91708f8f","pr_number":6954,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"172a674b0a38bb10e2125b5a2867d55ded44654f7e5e154155ab760de9601ea3","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T23:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11eea31d1d40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\n    let _endTime = datetime(2026-10-08T00:23:30.3337686Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-7nt846zks6zp'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T00:23:37.3079994Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\", ActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", SubActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityStack:\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x11eea2518d38\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x11eea31d1d20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x11eea31d1d00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x11eea2cd05c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x11eea2cd0560\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x11eea1622000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11eea31d1d40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\n    let _endTime = datetime(2026-10-08T00:23:30.3337686Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-7nt846zks6zp'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T00:23:37.3079994Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\", ActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", SubActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityStack:\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x11eea2518d38\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x11eea31d1d20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x11eea31d1d00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x11eea2cd05c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x11eea2cd0560\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x11eea1622000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107969278706716672","occurred_at":"2026-10-07T23:00:14Z","signature_id":"666d734da3ae47fa83d61d5609163ffa16d5c7b7a03ff4c22bce8466e9417d11","pr_number":6928,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e5bf9e749f8737ec65673ce5aaf89bc324878bae903ebb6ec2b0453ffece29b8","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T13:01:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x119e8d352a00\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\n    let _endTime = datetime(2026-10-08T14:29:33.7915314Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-7dxbsk584tbw'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T14:29:42.5063608Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\", ActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", SubActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x119e8dac0f30\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x119e8d3529e0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x119e8d3529c0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x119e8d36b280\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x119e8d36b1a0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x119e8b843000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x119e8d352a00\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\n    let _endTime = datetime(2026-10-08T14:29:33.7915314Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-7dxbsk584tbw'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T14:29:42.5063608Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\", ActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", SubActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x119e8dac0f30\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x119e8d3529e0\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x119e8d3529c0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x119e8d36b280\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x119e8d36b1a0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x119e8b843000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","occurred_at":"2026-10-08T13:01:55Z","signature_id":"5b0ce0129345f929b9f05e55a452f600982aefa185554902a0b82bcf5372be48","pr_number":7387,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1b9753d6df9613da46b4b9cfaf32730b876169734288bd96a4ec92c7f1a772fa","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n    let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-82f6cfwmh699'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x258297402990\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n    let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-82f6cfwmh699'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x258297402990\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-82f6cfwmh699'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"670cbec864bb9c486cb9441591ed8f63fe3036fd2cb244750889987cfff959c9","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T13:55:41Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-07T14:40:32.913Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-10-07T14:40:33.412Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-10-07T14:40:33.412Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1\ntime=2026-10-07T14:40:33.440Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 namespace=multicluster-engine\ntime=2026-10-07T14:40:33.491Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 dryRun=false\ntime=2026-10-07T14:40:33.491Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1\ntime=2026-10-07T14:40:33.534Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1\ntime=2026-10-07T14:51:01.640Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 err=\"failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-07T14:51:01.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107832305794945024","occurred_at":"2026-10-07T13:55:41Z","signature_id":"5313fb62a7a2ad329b1783c26de868383dd6a90d3fce29c37b41985c964d9b72","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478","failure_pattern":"failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-08T02:48:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","occurrences":1}],"full_error_samples":["time=2026-10-08T03:31:09.066Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-10-08T03:31:09.688Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpci01 kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:c5d026c4a8335c2d098ce04776e25babc3e7773e70817b5b709f388bb52488d2] metricLabelsAllowlist:[nodes=[node.kubernetes.io/instance-type,kubernetes.azure.com/agentpool]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[system]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:9c285c83d939df7d0bdf27287562bb6fcf8fd6ab639d899abba1f14a0c2d5647] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:5fc5d09d0245cc1e7c6432ca9b0ebcc652c047b550d0289ea45725c14343585d]] thanosImage:map[registry:arohcpsvcdev.azurecr.io repository:thanos/thanos sha:b567818fe608067eb0f1d7c2c4fe361e7ad83c8a256234c97685f1d0bf670cc8 tag:v0.41.0] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[serviceAccount:map[managedIdentity:0b00177b-a54e-4160-b95f-d8ba55b89f42]] prometheusSpec:map[externalLabels:map[cluster:ci01-j0275840-svc environment:ci01 region:westus3] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:7ef9c1d80e4e8e1eec1a207166af22930c8f11200f25001d1a3285b30b4f13f9] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-ci01-j0275840-svc-ng1v.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-d00da46377ca403ab27fbe4e13d51efe/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 replicas:2 resources:map[limits:map[memory:NONE] requests:map[cpu:NONE memory:NONE]] shards:1 version: zoneCount:3] schedulingRole:system]\"\ntime=2026-10-08T03:31:09.688Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:31:10.214Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-10-08T03:31:10.584Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-10-08T03:31:10.584Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:31:10.783Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:40:20.024Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-08T03:40:20.024Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108026812570275840","occurred_at":"2026-10-08T02:48:51Z","signature_id":"6bac7c9167143bb4a1dfabfc4a6d81083bef0f715dcabad843cd680ef59999d4","pr_number":6775,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6","failure_pattern":"failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-09T14:02:38Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-09T15:06:37.853Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-09T15:06:38.448Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 values=\"map[captureMode:all clusterName:ci01-j4897024-mgmt-2 enabled:true environment:ci01 image:map[digest:sha256:11b789c31d680e7f988ca6ecb8901408801a582f560ea82572d969f5ddb243a6 registry:arohcpsvcdev.azurecr.io repository:ci-op-crcb6znc/pipeline] region:westus3 resources:map[limits:map[memory:256Mi] requests:map[cpu:20m memory:64Mi]] serviceAccount:map[name:swift-recorder]]\"\ntime=2026-10-09T15:06:38.449Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-09T15:06:38.710Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 namespace=swift-recorder\ntime=2026-10-09T15:06:39.018Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-09T15:06:39.018Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-09T15:06:39.140Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-09T15:16:42.762Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"\ntime=2026-10-09T15:16:42.762Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","signature_id":"24872b7f19e6b96289ddd32696a00244aac5511dd32a2c2694b7aef4a036c3f7","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b3e3dde3ed6e86eca14ea52f55b0d41a64a9d455bb9f2e88e86dfa581de65743","failure_pattern":"failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","occurrences":1}],"full_error_samples":["time=2026-10-12T10:17:07.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull description=\"Step acrpull\\n Kind: Helm\\n\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:b0cbba7044d5142f8545758e5a69b7b35e7f8c6af596bc972e164d2b345978a7 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.589Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull namespace=acrpull\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull dryRun=false\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.976Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:22:16.134Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-12T10:22:16.135Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109576437055885312","occurred_at":"2026-10-12T09:26:31Z","signature_id":"5ac3f50227d6a7076813d8bbbe9de0e8c8876a26e6800117bb2654c1677cfb10","pr_number":7466,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-28","2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-07T18:42:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","occurrences":1}],"full_error_samples":["time=2026-10-07T19:22:31.326Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-10-07T19:22:32.066Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values=\"map[enabled:true image:map[digest:sha256:7c0c84fdd1dde7d32754d602c60e7432108ecef850d5bcd2efc549110fbda458 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-10-07T19:22:32.066Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-07T19:22:32.374Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events namespace=monitoring\ntime=2026-10-07T19:22:32.684Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events dryRun=false\ntime=2026-10-07T19:22:32.684Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-07T19:22:32.822Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-07T19:27:38.910Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-07T19:27:38.911Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904302923124736","occurred_at":"2026-10-07T18:42:03Z","signature_id":"6b8d9984b55592851b0841c5e8c4a0f18f8f61520e7df4a51c7e6afc9a707725","pr_number":7361,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.3215434083601286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-09T21:37:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 651.764s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x104322d955a0\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[23:00:07.094]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n    }�[0m\n    �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n    }�[0m\n    �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n    }�[0m\n    �[37m[23:00:10.320]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:20.786]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:20.857]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:21.018]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:21.198]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:21.347]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n    }�[0m\n    �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[23:00:22.064]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n    }�[0m\n    �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n    }�[0m\n    �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n    }�[0m\n    �[37m[23:00:25.198]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:31.232]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:31.233]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:31.296]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:31.363]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:31.520]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:31.665]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:31.785]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n    }�[0m\n    �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-kubernetes-events: command failed: exit status 1\n    output: �[37m[23:00:32.265]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n    }�[0m\n    �[37m[23:00:35.416]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:41.611]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:41.684]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:41.712]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:41.839]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:41.983]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:42.104]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n    }�[0m\n    �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[23:00:42.585]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n    }�[0m\n    �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n    }�[0m\n    �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n    }�[0m\n    �[37m[23:00:46.447]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:49.188]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:49.244]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:49.310]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:49.455]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:49.591]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:49.701]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n    }�[0m\n    �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-skip-hcp-and-events: command failed: exit status 1\n    output: �[37m[23:00:50.121]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n    }�[0m\n    �[37m[23:00:53.109]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:53.110]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n    }�[0m\n    �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n    }�[0m\n    �[37m[23:00:53.335]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:59.389]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:59.462]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:59.614]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:59.757]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:59.878]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:01:00.172]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n    }�[0m\n    �[37m[23:01:00.173]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n    {\n        s: \"must-gather CLI test failures:\\n  query-basic: command failed: exit status 1\\noutput: \\x1b[37m[23:00:07.094]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.320]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.786]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.857]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.018]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.347]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:22.064]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:31.232]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.233]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.296]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.363]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.520]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.665]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.785]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:32.265]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.416]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.611]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.684]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.839]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.983]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.104]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:42.585]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.447]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:49.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.244]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.455]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.591]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.701]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-and-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:50.121]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.109]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.335]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.389]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.462]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.614]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.757]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.878]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.172]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.173]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 651.764s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x104322d955a0\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[23:00:07.094]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n    }�[0m\n    �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n    }�[0m\n    �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n    }�[0m\n    �[37m[23:00:10.320]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:20.786]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:20.857]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:21.018]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:21.198]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:21.347]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n    }�[0m\n    �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[23:00:22.064]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n    }�[0m\n    �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n    }�[0m\n    �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n    }�[0m\n    �[37m[23:00:25.198]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:31.232]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:31.233]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:31.296]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:31.363]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:31.520]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:31.665]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:31.785]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n    }�[0m\n    �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-skip-kubernetes-events: command failed: exit status 1\n    output: �[37m[23:00:32.265]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n    }�[0m\n    �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n    }�[0m\n    �[37m[23:00:35.416]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:41.611]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:41.684]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:41.712]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:41.839]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:41.983]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:42.104]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n    }�[0m\n    �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[23:00:42.585]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n    }�[0m\n    �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n    }�[0m\n    �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n    }�[0m\n    �[37m[23:00:46.447]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:49.188]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:49.244]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:49.310]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:49.455]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:49.591]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:49.701]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n    }�[0m\n    �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n    }�[0m\n    \n      query-skip-hcp-and-events: command failed: exit status 1\n    output: �[37m[23:00:50.121]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n    }�[0m\n    �[37m[23:00:53.109]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[23:00:53.110]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"query \\\"backendControllerConditions\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n      \"name\": \"backendControllerConditions\"\n    }�[0m\n    �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n      \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n    }�[0m\n    �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n    }�[0m\n    �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n      \"count\": 611,\n      \"files\": 1\n    }�[0m\n    �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n      \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n    }�[0m\n    �[37m[23:00:53.335]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n    �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n      \"commits\": 294\n    }�[0m\n    �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n      \"clusterIDs\": null,\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\"\n    }�[0m\n    �[37m[23:00:59.389]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n      \"clusterIDs\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n      ]\n    }�[0m\n    �[37m[23:00:59.462]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n      \"clusterNames\": [\n        \"ci01-j8117120-mgmt-1\",\n        \"ci01-j8117120-svc\"\n      ]\n    }�[0m\n    �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaces\": [\n        \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n        \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n      ]\n    }�[0m\n    �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n      \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespaceCount\": 4\n    }�[0m\n    �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:59.614]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:59.757]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n    }�[0m\n    �[37m[23:00:59.878]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n      \"cluster\": \"ci01-j8117120-mgmt-1\",\n      \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n    }�[0m\n    �[37m[23:01:00.172]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n      \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n    }�[0m\n    �[37m[23:01:00.173]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\"\n    }�[0m\n    \n    {\n        s: \"must-gather CLI test failures:\\n  query-basic: command failed: exit status 1\\noutput: \\x1b[37m[23:00:07.094]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.320]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.786]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.857]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.018]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.347]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:22.064]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:31.232]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.233]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.296]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.363]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.520]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.665]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.785]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:32.265]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.416]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.611]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.684]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.839]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.983]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.104]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n  query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:42.585]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.447]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:49.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.244]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.455]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.591]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.701]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n  query-skip-hcp-and-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:50.121]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.109]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n  \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n  \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n  \\\"count\\\": 611,\\n  \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n  \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.335]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n  \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": null,\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.389]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n  \\\"clusterIDs\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.462]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n  \\\"clusterNames\\\": [\\n    \\\"ci01-j8117120-mgmt-1\\\",\\n    \\\"ci01-j8117120-svc\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaces\\\": [\\n    \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n    \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n  ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n  \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.614]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.757]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.878]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n  \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n  \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.172]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n  \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.173]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n  \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n    database('ServiceLogs').table('clustersServiceLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n    | where isnotempty(log.cid)\\\\n    | distinct tostring(log.cid)\\\\n    | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n    database('ServiceLogs').table('containerLogs')\\\\n    | where timestamp between (_startTime .. _endTime)\\\\n    | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n    | extend logStr = tostring(log)\\\\n    | where logStr has _cs_cluster_id\\\\n    | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n    | where isnotempty(resourceId)\\\\n    | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n    (\\\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n        // Includes the CR reconciliation logs\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    ),\\\\n    (\\\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n        // Includes: work agent operations, manifest application, status feedback\\\\n        database('ServiceLogs').table('containerLogs')\\\\n        | where timestamp between (_startTime .. _endTime)\\\\n        | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n        | extend logStr = tostring(log)\\\\n        | where logStr has _cs_cluster_id\\\\n        | project timestamp, container_name, msg = logStr, log\\\\n    )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","occurred_at":"2026-10-09T21:37:50Z","signature_id":"cf36901b570ac1dee52b2e3df8bdc060e95f8945a01b9c43a720ca81ce35091f","pr_number":7446,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":10,"failed_runs":3,"raw_occurrences":7,"matched_occurrences":7,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":30,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["DEV"],"trend_counts":[1,0,6,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-08T20:12:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x25f97d5e2360\u003e: \n    resource group \"tls-endpoint-cluster-z5gqw56sgjhd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-z5gqw56sgjhd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"name\": \"cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T21:40:36.777928134Z\",\n      \"endTime\": \"2026-10-08T22:04:36.806096142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tarq2rh1nt9l8n2qiphckhb46c0kvk6 still exists (deletion dispatched at 2026-10-08T21:40:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x25f97d5e2360\u003e: \n    resource group \"tls-endpoint-cluster-z5gqw56sgjhd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-z5gqw56sgjhd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"name\": \"cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T21:40:36.777928134Z\",\n      \"endTime\": \"2026-10-08T22:04:36.806096142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tarq2rh1nt9l8n2qiphckhb46c0kvk6 still exists (deletion dispatched at 2026-10-08T21:40:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x63eda6241c8\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"name\": \"c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:08.268369953Z\",\n      \"endTime\": \"2026-10-08T17:27:11.496076999Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmuj8loas31r4p690qgf675h7c53q still exists (deletion dispatched at 2026-10-08T17:03:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x63eda6241c8\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"name\": \"c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:08.268369953Z\",\n      \"endTime\": \"2026-10-08T17:27:11.496076999Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmuj8loas31r4p690qgf675h7c53q still exists (deletion dispatched at 2026-10-08T17:03:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x317d01c40258\u003e: \n    resource group \"v20260901preview-9cxtwb94v2tg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-9cxtwb94v2tg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"name\": \"4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:49.176345278Z\",\n      \"endTime\": \"2026-10-08T17:27:51.502199148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmtqk63qrfodamveh24r6lp7p43kt still exists (deletion dispatched at 2026-10-08T17:03:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x317d01c40258\u003e: \n    resource group \"v20260901preview-9cxtwb94v2tg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-9cxtwb94v2tg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"name\": \"4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:49.176345278Z\",\n      \"endTime\": \"2026-10-08T17:27:51.502199148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmtqk63qrfodamveh24r6lp7p43kt still exists (deletion dispatched at 2026-10-08T17:03:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108289457667969024","occurred_at":"2026-10-08T20:12:31Z","signature_id":"4a13b0e3b632754c621cd1f40992e4f49a86b9b5a5adaf7a0291091d3c763eaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","signature_id":"2bccea3f5889cc8d014f896b80d99afeb5d988c9e408682319482453975203f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","signature_id":"7801a21b4f98263d138c84dd4d41be41bec31d075db9a5534f6fe7a256d4fe97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","signature_id":"afaf794a0f74bf5028bd0c8acfda702e8fa09c3b57b33581a270af808e618e1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","occurred_at":"2026-10-08T00:15:03Z","signature_id":"4687436a0bfcfca8d25b9c7ea62a3459ab48c2ddd70a922beb77b67c78af05b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","occurred_at":"2026-10-08T00:15:03Z","signature_id":"56eea831600730567aa0c1ce05d30226ca8a8d1e83b1c5e0f2bc75bde312d724","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":10,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T15:43:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:148]: failed to delete customer resource group \"cx-rg-cluster-9528856dtms2\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2df35190540\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\n    {\n        msg: \"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2df35ff14c0\u003e{\n                s: \"timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:148]: failed to delete customer resource group \"cx-rg-cluster-9528856dtms2\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2df35190540\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\n    {\n        msg: \"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2df35ff14c0\u003e{\n                s: \"timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","signature_id":"c37035f44e6baf132cd362aa9454d42aa8651a4d8c8d859449fd5863866a8dc3","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":45,"failed_runs":27,"raw_occurrences":280,"matched_occurrences":274,"runs_affected":21},"rows":[{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":79,"runs_affected":9,"failed_runs":9,"run_impact_percent":20,"anchor_occurrences":79,"after_last_push_count":126,"also_in":["DEV","STG"],"trend_counts":[6,43,36,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":39,"prior_last_seen_at":"2026-10-08T17:28:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x553d3fef80\u003e: \n    failed to create NodePool np-kn7h4c: failed waiting for nodepool=\"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\" in resourcegroup=\"rg-candidate-5-0-5tp4pjp8hk9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"name\": \"b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:41:12.213982763Z\",\n      \"endTime\": \"2026-10-08T19:10:15.712133624Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x553d3fef80\u003e: \n    failed to create NodePool np-kn7h4c: failed waiting for nodepool=\"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\" in resourcegroup=\"rg-candidate-5-0-5tp4pjp8hk9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"name\": \"b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:41:12.213982763Z\",\n      \"endTime\": \"2026-10-08T19:10:15.712133624Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35ae7eca0680\u003e: \n    failed to create NodePool np-ntb88n: failed waiting for nodepool=\"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\" in resourcegroup=\"rg-candidate-5-1-tvlhq8pw8r2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"name\": \"00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:45:34.339197477Z\",\n      \"endTime\": \"2026-10-08T19:14:37.105782951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35ae7eca0680\u003e: \n    failed to create NodePool np-ntb88n: failed waiting for nodepool=\"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\" in resourcegroup=\"rg-candidate-5-1-tvlhq8pw8r2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"name\": \"00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:45:34.339197477Z\",\n      \"endTime\": \"2026-10-08T19:14:37.105782951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3108b67364e0\u003e: \n    failed to create NodePool np-8mqkrm: failed waiting for nodepool=\"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\" in resourcegroup=\"rg-candidate-4-20-xmx872mb68nv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"name\": \"827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:32:58.020047255Z\",\n      \"endTime\": \"2026-10-08T19:02:05.605513431Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3108b67364e0\u003e: \n    failed to create NodePool np-8mqkrm: failed waiting for nodepool=\"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\" in resourcegroup=\"rg-candidate-4-20-xmx872mb68nv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"name\": \"827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:32:58.020047255Z\",\n      \"endTime\": \"2026-10-08T19:02:05.605513431Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"1de706bd8c752817d789e3f2ee0485ef15ce188abeabb1b4b45baa1c1e902769","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"607737456265c716166bec4f750288f0594652400fdb685dc45a9ab0c9b032ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"8360fc4f2965ead66c616d75f5409d9c8aee0164b8d4d71a8ba9d6db0fc9f8aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"8fae7edbf3fe4eba4c8d3ad2ab08abf901803a5c7eb69c8c4b81e4c6355c2ac1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"9fc1a6a4fa883fc6875002b44e541ced39ee5185ee8b663d0df77575e08f0939","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"cbd310b6320cdff663edcfcdc2aaf12bce0f77d46d8accdace4e10db5e71773a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"e57178da6231f676c3543909d6927a0653c8f12c71f9a8924178794a486457a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"eae823e6b74a95a2b1e84a019161309a32d30986fbaa5b35e10457bb32383c4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"524a2a76e90c2d1da53a5c0a9582d861268831f07109dc1e7db02e6ed4141945","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"5f3bfb947330e190172ba5110d4435e2bf0388db1bc378c5cd2b5067c49ec984","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"77f4bb331d7c87c8c6f645d4b5f13f08407295d66ee31371666f5da75544d885","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"78b9b25829be4c3984fe03924f28bc2a19d7a1e5fbcf3047833b3465f0d57419","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"9b452c1be3c93f975397b5ccfaabffaec5e3ee43b442f86ff7704c7cfc6c3d23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"ca182933bc653faf7eb30127aebd6ed595c5e97059da875779383f3b49535773","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"ccc88a84bcfc47cb169e8a095ef78886581bb9c45ae50908d9d1797d28040551","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"cea1f06c65c9ac1a5240a3f2299004282ac5955daa892019aaf1e458ea109d8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"d7c2cc02f3a607ef7b6cbe3b49264471171e7681498d80954c9331231b7a8a89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"0e5d065bab30d941a1416295359f6c50ec9d353a91602de79cd9253e6bac9e86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"212d4cea3adecaa7fb0b5b48ed66e7c9510a248ab4a2550d0e77537380105fa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"50a678b577dd6ec95a39c4930784d2f2f8df4c2e4deb6c817d27050680bc7bad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"704461f11fba58d3264f2375448ec8d0dcfefed9be4b8d25b1805dc09d5f9d85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"8d541c9f52f89327eb524d55f2cbc5184388fedfbf5f16fa8cb7e24ae40f7f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"b9836bfff02572891050840308d6c8bb9be4b2b018e321a4135572ee218da36e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"bc2d06f2e3bbb11b62e2bdbb0371b19a15817c0acbbe41a8a602481250c62016","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"c07cc1511d492c4af5906301b6ac8729ea5b2acd7f223c5094e456cf7e259b7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"ca5e81422055ee4bd8d2f8df0fa691b5dc11cb7cece132e6083770e89a70662a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"f6915c2acdb65b50b5f491323caf68c80329ffbf00cf123f661b2d3be53776ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"286f2b337db1c06afe4f552fafe360d2df35c905db51dc5712e0039ace5c0cf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"363bec732de5d54b45f82f6c6410eef0cdb5ff0af00c3fc2788bcb78b1e622e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"7bc1860acf1bdc160e1d9d3e083e6a77b4a25446ec861b06bce73c5aef74c5d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"8dbd91ef71ec969c7b63355f72e7f62a344f89bc68b893cbfcc89d6c2e547cb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"8eb81065221254a41909e446e130e2c5564d04ecbae268b12a83dd5adbb708e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"97e9cfb4d9ae6fd232412678c19b7ea57ae3f6d1c4164518dd3f623a9673d2b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"a2308670e8d0cf661e1d2de3f880177522e71068292186eb5069f5861f9c634c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"d9b09315f5cbffbd9b174578ef412adca169d6ebcfe103e2e14792ace908df4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"dc21971b0a431404b75ca2aa3a8a2071c2727c9e220c888a3e332140a7c42a55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"016fceb8857335fe4989864c4eb9b68117f9071b521a6c986d09b960f5ee4cc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"0df775ba92680196995c3176cc7ea2cf943ff930e220b5800c510ebe552cb1a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"2a467c8dce60d0e129d691c98b8512c8ec1ed634b0dd9e1fd39a756686e7121b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"4e1cdbd87ab7f2945023de5abed108f7e637f867d4aa7849637d211a4e777718","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"50f6bef9fb00580c65f782fcadba664ddc8335bf13557695ef58074a611962dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"89dcd307570448e824d023284861443a0dfbff7e5115cf1259f0fe9ffd4aa969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"c77f01f179024436a11b7e008e5544b959c8dce85fa7b4baaf4c0b19ae4e63ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"c8c0700c36a2dd608f6a26fbbe6a7b358648287bea29f72e02d30d5eb97939a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"d9a1da04dc652deaf64a62fa6192a041922c1cfe9c39fc4d3a8476773a01a71c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"1b24e8be23bf1c4a3a07ea52d5db1e9347b68ceaa63a98aa205db1be1d9e27db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"2f0ca01949f4ede477fc7bb9d9a950ce013da9ddd6123baf58a79a544d4837fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"5ad64e3bdfb8fb37ebf569989c2484a58f8d9a29ef2b587e78f90bbd9aca0db6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"b15f13ff28e10dd36674e727784b787d32b868277a3a5254568d42d9dddd74c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"cea78ce35fc90771f705f79a4927fdf38597e343e521b69fc7526b14eb7ab849","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"de05f0b33c084620030e8c696e91ec71531528875543bab2b0545455a55994aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"f7d1fcc30d377fd476b9f57741967c7af1149a2007cd215bbdac2b9536cded49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"04ad7125fe4905ae031c4007581e188577bf5010611a020b6a2efc132c5dc319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"753d143e88447561451de08944a9408e6f40d29e02738dd21a3764fd6cbe715c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"9785d8491224b970db4521504142e02625dbd60d73aaa543ecd4460be888e210","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"b80fe52aacb5620760c6bea3b3ed05432ded767100eb2b21c6c1e752e2a3acc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"bb8fa03a2e39c2209193639a0d46220b87e342b6202a3ad16c83d99f1ef37edf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"c7f5bdc4ebc923e20233f949e36f2ccb4fee458a453a1ee7a629440470bf56a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"cba8f12de6a543925775d61a4dc9325c74c4f0c58a30cea6a9fa01d63b96e361","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"dd2af2d4c434317800a4e02b03119effcc044182d265a3a22bcae48911b84b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"e3fb4a2b49a31e56c0902493d1ea2275a273a2c53bab17b6f360121094551afe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"522a7104067322270c1dd86403e8358453c5864b0e27aac4cc5b395bb3bb08b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"5cdf839fb7474b7c4643969bf85e742d6e7999ab72dceb75b236195841c8a7ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"92a56bbeee063c237c54c6a4c494c87e10a3cc5bf6ede5a63dcdfaf9e7e689fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"987d0ce54e6e4418fc185183edfbc093147759ab1307a6f8a59df037a4923be7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"9a80126b88b88f73aaed428e993ee3eb35ad2dc5ae4652821263df2cf667dc44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"d79148cbc396e3fb2417bb6a5c08546a4bc7626d2cb1d2b265e599d0466ee183","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"e8b902ffd47c497bcda3d3146c3d7891d7784d3563fc15636c6894b1f96e87aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"f60ae0ea29618f99864000653c981584b53aa5f616efd1c3c85d5021999710ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"fbd0c561558b517a4ab86f8ad9b18ffae8f602f9d4cdc462cbca34d38dfdb848","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"1ce8678d638300e56c4666b5590395913aef8694ae27b3df675cb7ad0f7b01b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"3f3489b7375ffb121a83750f4359bd32c3f7d88aea2c87381d44bdbba593595c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4116ee0556c26bd8833a3374c74c6a16c11f1c9d2170f9475869ca6784808061","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4af35e4cc7af67899aa17ed379f7e70c8cb4b98d18356ff405fe1baaa2a79cc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"6ce961a4bc8897fbd7b98df3cc5a58955ed5e747b18252fdfef55c7377a098c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"7f76fffd8b4b60d1d840bb53756f03a0b49f3555b816311695931c4a85b6abbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"ab88004ed09b9059f4902caf266cb767ff949a7b11999ba22a96ad8310ee0f0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"c29e28b214fa390074f60edb54d46b123550e1ba11cc9a9e14a2585e50f1d30d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e915fc8a853ff7f35cb4923c9b73fd715420c8f36391d017033ce5747e871487","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":51,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":51,"after_last_push_count":51,"trend_counts":[0,51,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z9xhpt9dxztr/cluster-candidate-4-22-8frl8d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc3c4040eec0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8frl8d: failed waiting for cluster=\"cluster-candidate-4-22-8frl8d\" in resourcegroup=\"rg-candidate-4-22-z9xhpt9dxztr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"name\": \"6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.12350471Z\",\n      \"endTime\": \"2026-10-07T11:13:04.942313802Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z9xhpt9dxztr/cluster-candidate-4-22-8frl8d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc3c4040eec0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8frl8d: failed waiting for cluster=\"cluster-candidate-4-22-8frl8d\" in resourcegroup=\"rg-candidate-4-22-z9xhpt9dxztr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"name\": \"6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.12350471Z\",\n      \"endTime\": \"2026-10-07T11:13:04.942313802Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2851e862a620\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ws854cfjbfc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"name\": \"5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:27.70826866Z\",\n      \"endTime\": \"2026-10-07T12:10:34.84968826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2851e862a620\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ws854cfjbfc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"name\": \"5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:27.70826866Z\",\n      \"endTime\": \"2026-10-07T12:10:34.84968826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231265410c40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6pl2gptcdfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"name\": \"3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:27.398564047Z\",\n      \"endTime\": \"2026-10-07T12:08:34.791319585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231265410c40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6pl2gptcdfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"name\": \"3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:27.398564047Z\",\n      \"endTime\": \"2026-10-07T12:08:34.791319585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"03da4f253bd1559e010815404447df7a21bfdb927bb20e898538689c3b1a7fc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"14135c0f42e6038fec848d3d13ac98a506c6bb5e44786249455cbf6831ee7ee5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"161f29459dd25be2b9948cb9b2928ee7923fcc1e8cff6ecba8619d7726e0f54d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"1882d241a3c9a7844d4783366ce6767db79330aaad9fc90ff0cbd2d6aa042b9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"222845611c5a37be3043cb1e73ff7a15f6703e7a0b75ca16f7f6d42711571715","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"22d4e8be87148da939695d13e1247fdbef5bf1728991abf73a2144aa28fb21d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"2a4544a43b4c4906031ddca838eac98d21615c334e36bf9f03833d3a9649ae82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"353b34662f60e8180bd22425317cae7ac8cb19040c048446d6da34eb0770224d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"38424e0b4e68892bec2eb668127066c6f2a26c76ecd81204afaf71f81b811e31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"3d2085352b10228c638ccaffe94249943e09d92267a10ea6e8a6f5acbc29dce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"3d70d948708128afed2253c50f9bf1c308631c17ba710f3522db7f18ef353d1c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"44595e30de386c5eb9f145b02f8c2bb5d743f010820664d1bfb5aebca2d405ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"481b8b75a1b96ca5473b5e3428bf844583807c6c42f822464b4a107e600e55af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"50ea19fbe8e7d37bc39003918df0f3db3330d2f50721b75d7305c5fee9e0c335","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"5401c8357fce4c3d57d0b54d9d0c1f712605e1965171f2bc7a7b99c403d39d17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"65aa41b9d85ed7b6471c3cae1678d3b40aedc84ad18e6c8bd418d159be210f5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"677239a911859f072d4ad2996b92ed87c2c9b33ef1b746a6f2c25221940cd869","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"6bb41cca9dbaab1d3f7795fa9b64ec70faea8e4b1d05cd73381f7f45ac480051","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"76f0209f03661a65125740c9733c3d4a11d4682178d1f51fbb322ae760b9115c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"81065ecc30a88dbe3690b63b71b1ab98da072a79a5e00eafdac69a5cf5e2e363","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"85089d90025f7e4b136612cbe3126cdeb157f81d89a6d1f85bf84e1242f59f52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"8b0eef9c1f0b24176611911df5aa0d480ab4a2120a5657f9316084eca9ea0bdf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"9612cc0cf025f89e9420879dadf6053baa6863d7bb5e5eaf07f761e2805945f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"96927fc5b20d35c1c9f5483135274e1e4c184f0687b9c7836cf8e11f8d5b485d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"977fcb1eb57ae1e979b4c1b950f7d9cc8abf8782c740f514024b50110304411e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"9f87a1d4862080e4d4b05c636d70abf9d6119ecf1dc420ad540a69449802aba5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a031589172d1628bbf1aaa7c92834693d2ad406fbe2d747a22903fa0d686e2c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a0a734eda8f093e83a4d94cde244e3c5aa94a9b640ad6a9042a19b8c1730494b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a1712b1999aa403be90c2c39032dbf3482175f4135dafb21660a6280946668f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a2ce9a68f555401f462f07278b9c8957cd22840d85da390141086d71898ecd04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"aa67d6d32793cc52a0a7b7125b70972d054b6703e99dbe1985a2e305c66412a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ad1adaad6c931397a5cfb6188dbcbf7aec77f60cfab7d62edf2c9ea2f44522e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"b3c1cc9d18ffd5943aa3e4739d61d99e679706d5964b07a3b5374e971e51cc88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"b6c92859057095532f3f24d018d39c6d2961d7b695e023087e8f6547d3916117","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"b950484d8435e9ff010711daf067e8e3f2a694fad38348dda90d2c31aa435f5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"be2d920233867be6ec757af08451d598249bd0bcdda5bba21b83e5200e3fff3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"cec1415504f739ba8bb2e47520a81df319f676a287c871c00a9ca5c0bf800bf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"d514516b178028fea78395eeb242c18d910c33a636e520b12e6c30ec48120190","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"e6282f6819cbae90d2080bcc11689d8d54bca0907cf69387e75370c987b1244d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"e68e84c5ef68c6d5ed89d88a519e423d565bc4c0fe60d49c95bf93910fc3b5ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"e8b5cdeb0f741da0ec81fbee861c309ad3dc7a968fe5127637a89e53b5181949","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ea3cca5efa4a4d974d95cceb3e9779fac77cf63503170b6ca9e52bfc03e01dac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ecdadfc64076f4841dd66ae1fae57132eb211d25a58c181f0dbea68197273788","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ed9355468df05ca46faf44afb5f61337814d9c2ece35b6740bc25165e7398be6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f5875557a07249abfea079a118855841355a791d06199b1a53afdd74eb9ed76c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f62dfbfab386ba992afc971567b9d0d76b1b0e0237b5e0a68a1b3ec19e0c909f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f671365b45f055e398b3b3ea370acfc9dc7cbc2da9b1a10f010cc014e9b21b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f952fed5c3234700f79a41bdd08cf3c1afc8a1f474d83944941014db01bd9d40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"fee1f7575ffa79a5f721c4a55f609c98395d74d67ae1e00d333ecc7ff0426314","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ff16efcd5d70df29c345cf8295a63cf7efb04beaae9b28016e8a9a1d93d36082","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ff985ecaf6a3ec92db07f03052cef058d21f798d558e0c502a4afa2f5001396c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":37,"runs_affected":4,"failed_runs":4,"run_impact_percent":8.88888888888889,"anchor_occurrences":37,"after_last_push_count":109,"trend_counts":[18,17,10,10,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-09T02:00:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-s865cq\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0x29e047bb9ce0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-84fl6x5gkgjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-s865cq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-s865cq\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0x29e047bb9ce0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-84fl6x5gkgjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-s865cq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-pwmrlt\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0x364694577140\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-bcxf7z6jvkdm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-pwmrlt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-pwmrlt\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0x364694577140\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-bcxf7z6jvkdm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-pwmrlt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-qldg5l\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x200557060510\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-hldpsj9c8292/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-qldg5l\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-qldg5l\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x200557060510\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-hldpsj9c8292/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-qldg5l\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"2f66d792165c52d205f1c6a611ab324b91f4405832fbf3e31394198c2a8e5ca9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"34397fe5abf712908f5423e76c41e9fb9bf115f7d4c93ff88752ee98026f7fa9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"589b0f77a559ee7a26153a17c6bf39d2e9b99a08197510cd4eaee21d45be7bf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"78ec029001c30998a4e1babdab79d48c3c17f2f252beb1cc1f2638c1cfdd5957","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"7a7082dfbd050591871fcb9c5978abf946cd65bce9ed32eda21da8403bff14ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"7b0aa1f88528b7aab6ebd43d0c8bc0c167b397681dde35e640fb0e5257495a48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"d0f6c4eb56b2416135b269c1aa7389148782dac649d937c3c1e7b19d1f13896e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"d4bf2a2be500d39b22377146543f50824f15b605e910ecb76b543ea51e8343e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"da1b16a2e6b7eedee000ee0af62d7085a036babbf6f8b626eaec2f300a3d2e39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"e23504d5c07814913d080ea69d0ded698df5a4d99822500032ec3653c9766294","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"3432fe66c4ed90923c24f3ae7f15c3a4bfc33f20470b8a1b2b57487830b17e6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"35790e776fa5175e4de78d7f6617ad98c96a8f158a3232735e601d4e5b778256","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"49bc5c5cdc05073630918f616f1b204bce3a50fc2a76296971ab18ebc0822591","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"4ea5e665e1441c36ffd5c3ff6c454b7210f8c32ad40846122aa42883cbc59638","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"66b6c637f092b0a1ae34328f0c47a9d939259c22ca517a9d0e2276a87a3b13b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"6e99dc03e140d021122b5f99a0b39eb90363b2a3eb418fc13a8bcf957465177e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"81166f4884fdcd117896960c8c39ef279bd4c30f7dfcfe06334faafd896c846c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"d9ecb82f70950229f902353084a6415477e147720c50fc7d251959e1cc792350","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"e64509f96f2f07ea596d1a6b68e536a246bfe0c47479f375bb73c60def84644c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"f0bf9d5656cd4403fe7ec557e934ab25ebc30cdf02bf25bdc5454a196c5801bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0f0a376071a48cbdd5611a1be04be8ee77bb833d681538cdad43fbb56b08abe9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"1ebec80e3d431198968e72632366b202903ff8ad592e0ab7fed494102db1b4fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"4894e3e856cbc512e113a81c6782b48eb1a23170d9613bd7146b82b50a5ba5be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"4fbf30b74782eb5ddefc289814bdb46d277c794e317bb41e0d934ac244ef4a6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6a259fc11a58d55268cdd102b80e33c281baa34d38f39e72cecc21167dfcf5fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"764d2f0ddbacda8f6566b981d59dc30f4bcdb0c97dd732f715407bf107cec2b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"8048cfa91100e6e8b8333a3eeebf145761ee4c1f1cf7dbde205df021632e34c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"f9758d5d36fc27455b6a21116c9b88ebed4012752a67056213ed40b7fb482538","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"1cbf26ee1e5ab30922ecc4509d85eefbb36e982c6ef1ac8f2183590c4796adf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"3b196b8c142b7de81731bf3edd13cc8231edfe7f5ca4159cff7cc65b94f4ec03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4cb77a8e2eb04e51d1e31b43fc33204e1adae88dabf9e8306d0b10cbba5bc76e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"5d7b068f725ac38a9e9614dfdb837b0fb5f432769ef58cb2757748484e960a34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"7011579edb7da9f880b9ea345fd4c99a061064cbde4affb3e7853dd76feb67c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9338ca7a8b3338ded2c268d7f0328d1f40dc0524254e97f8e79312c850e87765","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"cf3da2b8d691239f5c5c38c7c621ba3e4f4593ae21d6761ff8aa4453720a3201","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"f3a567f87a32f64da957df0785589b6bd3c47799e90deadbfe5bc1b927ed369a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"f88443bde247b05fc776f561d632239634a07c67d0f9534f9fadfe4591ff0ae7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":30,"runs_affected":4,"failed_runs":4,"run_impact_percent":8.88888888888889,"anchor_occurrences":30,"after_last_push_count":78,"trend_counts":[12,12,9,9,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-09T02:00:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f0cf085080\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-6zf46c\" resourcegroup=\"rg-np-dg-minor-6zf46c-hll6hfr44s9d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-6zf46c-hll6hfr44s9d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-6zf46c/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6f0cf085080\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-6zf46c\" resourcegroup=\"rg-np-dg-minor-6zf46c-hll6hfr44s9d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-6zf46c-hll6hfr44s9d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-6zf46c/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1508ab4d7940\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-lc4jfj\" resourcegroup=\"rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-lc4jfj/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1508ab4d7940\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-lc4jfj\" resourcegroup=\"rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-lc4jfj/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22aa54096dc0\u003e: \n    failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-qs6slk\" resourcegroup=\"rg-np-skip-minor-qs6slk-svmgzjwq29hn\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qs6slk-svmgzjwq29hn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qs6slk/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22aa54096dc0\u003e: \n    failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-qs6slk\" resourcegroup=\"rg-np-skip-minor-qs6slk-svmgzjwq29hn\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qs6slk-svmgzjwq29hn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qs6slk/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"22c5f8ad3e9edbc947e95262b2baf90209d9181c5fe0a567570ea7382201e9a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"3997c3e7b5d9bc914c8451d8312e28fb0c306a5f38d600fd0ce6f681823af251","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"4a276b7331de267a380c97189d1fcd50c4f0d29178edf855603c725af8fd356e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"5091085de46b6155ec8a80aa3a08e418ed8e2e45274e5a57cc44080adfe9031d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"957ef542fb913c7900eb599a22e4916d0e0fd5c25e4c722cbf8460aa0fd3dd33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"ab1b252cdf5012fbe93f692e1802a7010b7b1c0d6094b73f85e4facc9115996a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"ae372b6b9650d0b7fb76a731c6bb73e771d1ad2a2f403c8ea046a4810a89d911","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"e39234445903bc24ac0fa7548e2bc935524d10f1a9dce0f385ccddebb248cdef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"f3603a0b53eed9bd4dff440d7ea05c63ed18b5ef29f36ef1fb22c8746fbc2725","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"11fb9973540ad640cafcdd9a78df56a498b2cd9d499c2536d6c8e235215b9784","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"3d17f8d6652d176f544f7ddbab6bf98de82badadb74a026139b33bc1b25c0814","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"697bacd58389e5a2c004217397ccee804ff03a55b0df4328e472ad3b6538aa9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"759bd9f85e38e37d1860e839ceb34b5c137190cb65b706eb8146bc3e9eb17622","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"93c5f443b65f365a99e52dd59b3e31b6da3855ac65193c7fc2b456d85c0da930","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"a502c3c33e95830c6b055e8b63c99d465d434e71cd54d550ac664ae0376a0a52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"b8dc64cff9c94bedff3d6968dd7a64f15705facf6a41874e487dabb240ea1551","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"ddf416b60b13f8a7895320145ea311edeee5e97b0e336d70a0126bdfa4e1c5e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"f6fc53eb549f0e7f2f95375c5420737172608e9674edf0be196e6c524d2b9c9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"140752297cfc5196913f69b2f55f75602e169015ac1cf424a20aa497281b774b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"43468b889804001bbbe14bf435f20f53ae868359b8d38f9fb0a8ca86f38b3a14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"5f39b424b8f2fd154227943af3c6cc49910c69a6a961a7b0c48ca3a834eee8a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"0058e792d032ae53f5bca2e179ae1c409cdd5b3c2bb64768e1eb44b0951a5c10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4cec21a3bb1485fbbf2fb4c6307e0674539ae34b252d1fd61badf2ee1e7ee5b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"5be1160c2fa84fb7c2ea3eb2b520a681bba0ca7197c4252afbb61a475fb1097d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"7577f160ea754982b08b2f90867551e7ee2355fdac02ed66e99f5880127d9f79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"8696e2e81d9653753fa26ee12fc06824880c95acb852d196ef0af32ebd7fd0a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"8cf036c6672baf731fc796bc4ac50089b693a6a516c1216bf1d9e70f531eaf90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e68cdd216d0150edd847f603ef682998ad7b0271775710e0858f3f7f5b37530f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"f8f6ef862ba076aa00ce8d20e4b36cddbf5813529ac198d1dd60133ef734dda8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"fac4ce6bbb505620cdc94744fa17f94c3c1fa837421bd30a8671b97b8cca987e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":25,"runs_affected":6,"failed_runs":6,"run_impact_percent":13.333333333333334,"anchor_occurrences":25,"after_last_push_count":79,"also_in":["STG"],"trend_counts":[2,2,5,5,5,4,4,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2a4f44e2d020\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gwxj7rx975nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gwxj7rx975nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 28bc07db-2472-48a0-b8ec-f57404d49246\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2a4f44e2d020\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gwxj7rx975nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gwxj7rx975nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n            correlation ID: 28bc07db-2472-48a0-b8ec-f57404d49246\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xb63c5f25ec0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ffrc4jvplr5w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ffrc4jvplr5w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 85fbf18c-d76e-44a1-a17c-93587336d09b\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xb63c5f25ec0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ffrc4jvplr5w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ffrc4jvplr5w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n            correlation ID: 85fbf18c-d76e-44a1-a17c-93587336d09b\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b32f7ffab10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-n86b7vqhp6m8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-n86b7vqhp6m8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 04e10847-e793-4a67-b207-a326ae8ddcfc\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b32f7ffab10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-n86b7vqhp6m8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-n86b7vqhp6m8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: 04e10847-e793-4a67-b207-a326ae8ddcfc\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"53bef87c15633c3f568f024da6cc38f582808a0877b3c985db197193b820c49c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"6b9a8d8d2e8c40d51b4cce49a4ede0535c9a9742ac1cde3633d2dbd66229d867","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"816302c6f773d8a3809afa480580f0992dc6bf52d93be454eb54c90fc89aac18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"f724161a4b388d34a0d7848bd2cb5190dd4142eda26cf33576fb2df82c2e7a7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"05c29ad480b5bf27166e88948710fee03cc002f61af8efde1454d88889eb8859","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"57f4b8a29534a1b97f7efd40c16cb7d3faa7d37950a5b8bae5e2ef89286cdd95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"85e4ca2686ea9b5410efbbf71ef6da967a97fc5a0f3281b695c388c6119013b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"cf7cb1ca2d45d1b8b3bddf2bf8a6201374e2153edb4dc1531f8dc0a98730aadf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109056606359523328","occurred_at":"2026-10-10T23:00:53Z","signature_id":"0c87fe2862b70392fca717019807fab807ac6a1df6d802020cbe6c6f748e60b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109056606359523328","occurred_at":"2026-10-10T23:00:53Z","signature_id":"69c9a01f3a64a41e9ac517c3bd76ddfa7690934fb41545b23fadff7db49831b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109056606359523328","occurred_at":"2026-10-10T23:00:53Z","signature_id":"a9390cb07411ccd99caca021bee2b3b48d605cde4d398cb8e073f2bcdb4a86b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109056606359523328","occurred_at":"2026-10-10T23:00:53Z","signature_id":"aa625ee288b29918e9cc08edf505cba9830a8d9e1679991737150b02018447aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109056606359523328","occurred_at":"2026-10-10T23:00:53Z","signature_id":"c3ceb0eb2868f4f90bc4168fd7bb5de7171337a193c041c9faa4bc2c7823af91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","signature_id":"028fb357f573ffa054d5144583f4c22cd50b81ab699f7eee531ee4cdeb8f5a03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","signature_id":"5f2ffe9a21529b96388c2140816f5b164f7b9b0de07aecad7e22bbca275aaa63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","signature_id":"943567e0e7942ccbf9ee1f42758dbeafd6a415e8fd929d6f9edd74a6a92d458e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","signature_id":"b6f4654eb69ad45b39e2c965dcb1bb7725b86526f162a602d4de14c605a4bbec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","signature_id":"c3a144c1a4b30ab1c9c2b6f88c994e5695f256aa071d503e64b63c37d33f0d7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","signature_id":"538bc1de69b7243ea254b7eafc69d03d17567c9e64b692cd74fb0c6457d5cc48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","signature_id":"583203b05397b372a4354f60b57a0bc62bc0edfa59d540e09c922f53a5450367","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","signature_id":"948425aa0583e35245acd6dfa26e29e48e09e7387b7c14f662b8e630e477a7b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","signature_id":"a9cdc35fe4bd7f2b83c85c8090894982dc2b8c173bdc03255cf86269d22ffc5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","signature_id":"fc06ea34826977c15833fec7d2cf3a0aa68559a73c0a34b4f46fe7992c958855","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"68d4ce843ab001efd587a8e4a37b0a8261c4ad82e127c6e95cfebd9d1f8cff44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"b614f8950b85c4c4d2722a409ef292ea389bae31115c3e2b81722d59a537583b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":10,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.666666666666667,"anchor_occurrences":10,"after_last_push_count":25,"trend_counts":[2,4,3,3,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-09T02:00:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-g5xfmb\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2d844c21d830\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5xdvnklmwfq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-g5xfmb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-g5xfmb\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2d844c21d830\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5xdvnklmwfq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-g5xfmb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-t4k2nv\" on minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x9a9211d2ff0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-wdg9f8phhv6h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-t4k2nv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-t4k2nv\" on minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x9a9211d2ff0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-wdg9f8phhv6h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-t4k2nv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-kqzff9\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xe01aaa2bbc0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7pmn5ng5wrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-kqzff9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-kqzff9\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xe01aaa2bbc0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7pmn5ng5wrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-kqzff9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"06438279994666e4b2d4e4cda8087f66299b5a578c7b2cd13360e3d1df435524","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"7fed0e3251f5b0df1a0ed777ac58de15b61f4ae1193d04b925b5765a33b1cadd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","signature_id":"d06c5ac4362d15796d9f2f41e12315787900a50fa301ee39c8a256c3850ceb32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"1609838bf5872d5e2d583f56af041bc1d204c93d24ccdb32dad101d6536f4ebf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"3979048c452a40f54f92711abf6015db7d4a3247bf1fce0134b9a35f9c83290f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"e088dc0de9a1fac4cac481a667a95c6c5ae11cc93acca3216cdc27fe06be74e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4aadc6cfa9574f8e73cbcf281be4f06712a54bf21ebdbda8d42084548d72bf1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"b4c938ffd1e07317c7f1a8b66bdd6168fa2ea345151308900639a09f23bc84fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"bbea7e2da871e62baa803b44294e1265be218d3de0dc6cf6f671b9fcb2dfd871","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"eca13765fa9405f95252cf15d7ab0f204511f501fe8006308939ad15b76113e0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":9,"runs_affected":7,"failed_runs":7,"run_impact_percent":15.555555555555555,"anchor_occurrences":9,"after_last_push_count":11,"also_in":["DEV","STG"],"trend_counts":[1,5,4,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-08T17:28:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2f509ecfe3c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bw4mbhgcm827\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"name\": \"d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:07:59.549473703Z\",\n      \"endTime\": \"2026-10-08T18:37:05.482678031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2f509ecfe3c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bw4mbhgcm827\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"name\": \"d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:07:59.549473703Z\",\n      \"endTime\": \"2026-10-08T18:37:05.482678031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3c50440e0018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nkr4c2vp869x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"name\": \"5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:08:57.323157264Z\",\n      \"endTime\": \"2026-10-08T18:38:05.479695963Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3c50440e0018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nkr4c2vp869x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"name\": \"5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:08:57.323157264Z\",\n      \"endTime\": \"2026-10-08T18:38:05.479695963Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6335433a090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-tpwrtzjcnmjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"name\": \"3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:27:33.571990767Z\",\n      \"endTime\": \"2026-10-08T14:56:35.04864797Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-8swjn-292k4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-8swjn-292k4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6335433a090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-tpwrtzjcnmjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"name\": \"3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:27:33.571990767Z\",\n      \"endTime\": \"2026-10-08T14:56:35.04864797Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-8swjn-292k4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-8swjn-292k4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"30be14ccfca5a103a44375ac3df669c521021f4c93308b56a32de7bd14926922","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","signature_id":"ecfced95d3fefee4d4fd692bafa6b115f8cb1c09453e7e1b0e30620a04bb8554","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"04197c3f7c272281d2f16a127440b51aecd5581858f715b36eb52e6c706ac7dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"399ce0931f87760834405b1f91f2bb0dc5c6dee722c1e4c00e390a98ef2e93ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"aaecb179fa0c8619e61820f2147630fb08f8bc80cc1c2f7237571a27bd0b49f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"d5adfea4d2420279717d9c7a8651e85415083eda53a1fc1cec731d467be31793","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"42cc3045165e80f27d068734529151281e6d04adb423b2dda7c90a129047ebf7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"02820abf0337fcd30dd807581ed2e7cced4ca7f03dbbf80f30a8a902b7db9b1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"45693c845a913fd4511bed4e661c2d3c6bfa8a4cc05cb75085a24072104e134b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":17.77777777777778,"anchor_occurrences":8,"after_last_push_count":8,"also_in":["DEV","STG"],"trend_counts":[0,5,3,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":30,"prior_last_seen_at":"2026-10-08T13:52:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a6cffe3f060\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jct5pzjh8mxm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"name\": \"b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T15:48:06.632742492Z\",\n      \"endTime\": \"2026-10-08T16:17:11.778582607Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a6cffe3f060\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jct5pzjh8mxm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"name\": \"b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T15:48:06.632742492Z\",\n      \"endTime\": \"2026-10-08T16:17:11.778582607Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d67242a00\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"name\": \"1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T05:42:55.903350034Z\",\n      \"endTime\": \"2026-10-08T06:12:00.742498982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d67242a00\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"name\": \"1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T05:42:55.903350034Z\",\n      \"endTime\": \"2026-10-08T06:12:00.742498982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2523ecbc0ea0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zgfh95c7gz2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"name\": \"cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T03:28:30.373260167Z\",\n      \"endTime\": \"2026-10-08T03:57:35.369426461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2523ecbc0ea0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zgfh95c7gz2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"name\": \"cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T03:28:30.373260167Z\",\n      \"endTime\": \"2026-10-08T03:57:35.369426461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","signature_id":"6771b69c9cd8b6f46bf94cd51d146bf6cecc95e92b532d2cb269b24721b30ded","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","signature_id":"e31120375fe20c3189a3f2c035d3f9c8bac3cf1283b6c17aad50ebf324b212ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"9f77071dc6747b0314cd8920ca2287311b939a55c7de09fa54be0bfe2bcd1f78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"ad8aa7d0e446ca97b82f47c75fb4959ef7171a8fc84389bd39c3be3472d4a550","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"f06515c3d076448409dc5ad1d9d943a2b21906ced720a3fc9685ab778dec06cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"19f8dd197db68f57fa6b4eeb4e85cec78926fb3e66586ba3c7b4f6152ea72984","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"29adca326faa1bc8f35a40a7651e6edc98aa900d6b8aa8c789d1b2db2ab468ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"dcfadc4ebeef20592def4e3d8be48fe15094c12bcb500f3186b13d44a79114cc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":7,"after_last_push_count":28,"trend_counts":[0,7,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186c1f59fe00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-xkrgtz49hrh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xkrgtz49hrh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186c1f59fe00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-xkrgtz49hrh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xkrgtz49hrh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e92b4ffb60\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jsrm64gc6pjs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jsrm64gc6pjs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e92b4ffb60\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jsrm64gc6pjs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jsrm64gc6pjs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1d4613307440\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-xccktj7qbpzs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1d4613307440\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-xccktj7qbpzs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"2a9264902cac7135ae10487055f2ec90d442d6082711c47d54f8a83eaf00eefd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"4285d36e41c52c52f14f33e255d32a9fbefe892fb4555c01f3990db62c8bb7f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"554cf56528558edb3b288f90dec934fc7fd2db28015f4ef2d61362bc08676deb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"67e3544022c911074f09ce65702282c6041cff3d54247562a71ba2e779489a2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"aa6f57794e5ffec707db8f8f3922b0d34c04ee60936df21bade7d3f757093a83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"d049db17883d8e9b0b2cefc43813652d974c416c9ec136721194aeabfe76194d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"dd8473cf389dd6f494be575174dc277ffa211b0aa8afa70458bd3be0e7b014db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.666666666666667,"anchor_occurrences":3,"after_last_push_count":6,"also_in":["STG"],"trend_counts":[0,1,0,0,1,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-10T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:175]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n    \u003c*errors.joinError | 0x2d289f100030\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2d289e7b3500\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2d289e7b34e0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2d289f0dae60\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2d289e6a6040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2d289e6a6020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2d289e168dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:175]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n    \u003c*errors.joinError | 0x2d289f100030\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2d289e7b3500\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2d289e7b34e0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2d289f0dae60\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2d289e6a6040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2d289e6a6020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2d289e168dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x3ac58974db78\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3ac589d3d2a0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3ac589d3d280\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3ac589190be0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3ac589e5f3e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3ac589e5f3c0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3ac58a59a640\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x3ac58974db78\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3ac589d3d2a0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3ac589d3d280\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3ac589190be0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3ac589e5f3e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3ac589e5f3c0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3ac58a59a640\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x2e5cf51ae018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2e5cf5120ca0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf5120c80\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4986960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2e5cf52a2040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf52a2020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4b086e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x2e5cf51ae018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2e5cf5120ca0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf5120c80\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4986960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2e5cf52a2040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf52a2020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4b086e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"74819dbc97c4d84b5230e834e8fd3d8a06bb306d8df4e0b9e906ef2406d3f4fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109056606359523328","occurred_at":"2026-10-10T23:00:53Z","signature_id":"37f9b3047f94391d0e2925e4e59eade02ef5df3a597fed499b4658df901bddb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"d4720f51184f2ea80269323a34de851f7131f4cdc47aa604ea02b7639bfb5d5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":3,"after_last_push_count":55,"trend_counts":[3,3,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":21,"prior_last_seen_at":"2026-10-07T23:00:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x312a48ea2980\u003e: \n    failed to create NodePool np-zgtwtf: failed starting nodepool creation \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\" in resourcegroup=\"rg-nightly-4-20-j42q4dvn7t74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-j42q4dvn7t74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zgtwtf/nodePools/np-zgtwtf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x312a48ea2980\u003e: \n    failed to create NodePool np-zgtwtf: failed starting nodepool creation \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\" in resourcegroup=\"rg-nightly-4-20-j42q4dvn7t74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-j42q4dvn7t74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zgtwtf/nodePools/np-zgtwtf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32c6fec798e0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-vbjww8\" in resourcegroup=\"rg-np-skip-minor-vbjww8-9dlhkbhpd9hb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vbjww8-9dlhkbhpd9hb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vbjww8/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32c6fec798e0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-vbjww8\" in resourcegroup=\"rg-np-skip-minor-vbjww8-9dlhkbhpd9hb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vbjww8-9dlhkbhpd9hb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vbjww8/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1829881ca3e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7pmxvz\" in resourcegroup=\"rg-np-version-upgrade-7pmxvz-jjjscx2plvvj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pmxvz-jjjscx2plvvj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pmxvz/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1829881ca3e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7pmxvz\" in resourcegroup=\"rg-np-version-upgrade-7pmxvz-jjjscx2plvvj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pmxvz-jjjscx2plvvj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pmxvz/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6037211c53fad4b43e675ddfe0a29c9bf8bd819adc8e5ef742dd4a3d5eac5fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"84b2c7e168966ee41c9321960df1699f3c7e6ff0bbf13ad9815db6fc66c666f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"95caab3bc51d32ccb091310bdf33b3e84812faa9fc5c890de7a18f95ac7e465e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":2,"after_last_push_count":6,"also_in":["STG"],"trend_counts":[1,0,0,0,0,2,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x11944cc8930\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x11944978520\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x11944cc8930\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x11944de5c00\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944de5be0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11945431e00\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x11944978520\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x11944978500\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x11944c97ae0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0x19c6c82ad0e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x19c6c78ffea0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c78ffe80\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7ea8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x19c6c8695740\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x19c6c8695720\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x19c6c7b65c20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"5bfccd42c01d56d5347db2bbce6e7db03d40121dacef58a1a5dc37e3bff7a16f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109418786846150656","occurred_at":"2026-10-11T23:00:04Z","signature_id":"eb1c33da48c2dd7e1a17df691460f82388ce7033187824a4224223b7cbcc8fe1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-q7jjp7\" with version 4.21.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x298b49702ce0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-q7jjp7: failed waiting for cluster=\"cluster-zstream-4-21-q7jjp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-jd6h7xk8btv9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n      \"name\": \"c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T14:02:09.836072198Z\",\n      \"endTime\": \"2026-10-12T14:31:10.061357256Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, cluster-network-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, openshift-route-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n            correlation ID: 414509d7-91c9-4911-b025-fd41c0e067b1\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-q7jjp7\" with version 4.21.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x298b49702ce0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-q7jjp7: failed waiting for cluster=\"cluster-zstream-4-21-q7jjp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-jd6h7xk8btv9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n      \"name\": \"c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T14:02:09.836072198Z\",\n      \"endTime\": \"2026-10-12T14:31:10.061357256Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, cluster-network-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, openshift-route-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n            correlation ID: 414509d7-91c9-4911-b025-fd41c0e067b1\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109630653069791232","occurred_at":"2026-10-12T13:01:57Z","signature_id":"625fe3d4a7bcc05f82b6be9c1b459cb3509760d899d8c1e3dcb5cf92e4d63ae3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-04T20:27:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ptqdgd\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21d67b55ea20\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-ptqdgd: failed waiting for cluster=\"cp-ystream-upgrade-5-0-ptqdgd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dznstqd64m54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\",\n      \"name\": \"9d14066e-1629-48f8-aa68-f556c4fd7044\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T07:51:24.545231929Z\",\n      \"endTime\": \"2026-10-12T08:20:29.935309867Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n            correlation ID: 7a60ad0c-37f6-45ef-98dd-efdfad687d6c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ptqdgd\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21d67b55ea20\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-ptqdgd: failed waiting for cluster=\"cp-ystream-upgrade-5-0-ptqdgd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dznstqd64m54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\",\n      \"name\": \"9d14066e-1629-48f8-aa68-f556c4fd7044\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T07:51:24.545231929Z\",\n      \"endTime\": \"2026-10-12T08:20:29.935309867Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n            correlation ID: 7a60ad0c-37f6-45ef-98dd-efdfad687d6c\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109535177171537920","occurred_at":"2026-10-12T06:42:34Z","signature_id":"fd4fc621d01f385247f810320f7e86a4ebf9edde1b95ddd444f5031ee94b4826","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x151c97ead140\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-p86tvztrm7hd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n      \"name\": \"01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T01:36:23.272806003Z\",\n      \"endTime\": \"2026-10-12T02:05:32.513113167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n            correlation ID: e3c8027d-936a-4cfa-802c-eac5497d3369\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x151c97ead140\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-p86tvztrm7hd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n      \"name\": \"01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T01:36:23.272806003Z\",\n      \"endTime\": \"2026-10-12T02:05:32.513113167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n            correlation ID: e3c8027d-936a-4cfa-802c-eac5497d3369\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","occurred_at":"2026-10-12T00:05:13Z","signature_id":"a33be23ec9f3ec13df877ffe26bd388352f59b753cee7526ea22101141d231fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"87227a922c90e8b74b9cd5f3e1315585fd49c7030b59796956bc7a208ed9e9d1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38da2adfeb00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-kbw6v2d2bhz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\",\n      \"name\": \"4899c274-4471-4ab9-871b-22368aa84095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T00:35:15.167750325Z\",\n      \"endTime\": \"2026-10-12T01:04:20.403308122Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-fk5wl-zdgdp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-fk5wl-zdgdp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n            correlation ID: 79b16636-2655-4af8-9ac0-f8e4eda0ad35\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38da2adfeb00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-kbw6v2d2bhz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\",\n      \"name\": \"4899c274-4471-4ab9-871b-22368aa84095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T00:35:15.167750325Z\",\n      \"endTime\": \"2026-10-12T01:04:20.403308122Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-fk5wl-zdgdp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-fk5wl-zdgdp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n            correlation ID: 79b16636-2655-4af8-9ac0-f8e4eda0ad35\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","occurred_at":"2026-10-12T00:05:13Z","signature_id":"ca3816013b61a6551e43b1071dcaa2fa8f2e2f088ce6dca69491ec625e587d83","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8da18459eef6017d12822e9dd3b87c09fdb695ab8d9cb4881da243c125e2db6b","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T02:00:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a6c3876d160\u003e: \n    failed waiting for deployment \"mi-missing-mi-hcp-cluster-8qb82d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-8qb82d/operationStatuses/08584101764787793835\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a6c3876d160\u003e: \n    failed waiting for deployment \"mi-missing-mi-hcp-cluster-8qb82d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-8qb82d/operationStatuses/08584101764787793835\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","signature_id":"71ce598d6d9d958fde2cd619eb779c67cc6e323a3cce0328ebb3cc5eff50003c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"86bebf5e619b1e97e49719a882eec1e5a32f3850fd6a0660d27a34314e24e439","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-11T22:05:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x5325624e738\u003e: \n    resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n      \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x5325624e738\u003e: \n    resource group \"cni-cilium-g76pf6t9h842\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-g76pf6t9h842\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"name\": \"179cc551-21d3-4583-b554-f672e37b3a84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T22:50:53.504973281Z\",\n      \"endTime\": \"2026-10-11T23:14:57.954117538Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tcruuiers9mtsh6kof4evdrtdk9p3e4 still exists (deletion dispatched at 2026-10-11T22:50:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109405118775431168","occurred_at":"2026-10-11T22:05:45Z","signature_id":"7581ac8d1a8bb671ccd2233ea3ec2804c211a820c9531c5cee88d22d89400cc4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8effde2aab4f5beaedda3d8b16dda29c381b8de0bb2c433491a2eb475fa67fb2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc79feec4060\u003e: \n    resource group \"np-autoscaling-xtg8j7kg4fwz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xtg8j7kg4fwz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n      \"name\": \"6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T02:47:19.509292457Z\",\n      \"endTime\": \"2026-10-12T03:11:19.252244159Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc79feec4060\u003e: \n    resource group \"np-autoscaling-xtg8j7kg4fwz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xtg8j7kg4fwz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n      \"name\": \"6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T02:47:19.509292457Z\",\n      \"endTime\": \"2026-10-12T03:11:19.252244159Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109448556992532480","occurred_at":"2026-10-12T00:58:22Z","signature_id":"92ca5810cc30b4aac8e0fae2571490fb634c25c0c1d6fd8a60f745c71eb4e4ce","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6414ac0ffcaccf7efef39a38c0b45dd9d34701a344992b34e8e112eb6e3a27b7","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T23:33:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x99270993880\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x99270993880\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"1b1d10ece3b90ada6d195629824ea959318da72c37d7910b78e8868c3143e501","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.2222222222222223,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T09:23:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2fbb237581e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kc94rh5nhxb8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2fbb237ab310\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2fbb237581e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kc94rh5nhxb8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2fbb237ab310\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"6a9c2523f9881cd28e189784f3244e2b462a82a81d6d904e71798161eadd2b44","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":20,"failed_runs":14,"raw_occurrences":114,"matched_occurrences":112,"runs_affected":12},"rows":[{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":50,"runs_affected":7,"failed_runs":7,"run_impact_percent":35,"anchor_occurrences":50,"after_last_push_count":114,"also_in":["DEV","PROD"],"trend_counts":[18,40,9,1,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":34,"prior_last_seen_at":"2026-10-09T05:19:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13cc8f58b420\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-sz5gdnfzqhdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\",\n      \"name\": \"2c72bb6d-878b-457e-bf39-56f98851abda\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T06:39:07.400427011Z\",\n      \"endTime\": \"2026-10-09T07:08:09.774067703Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13cc8f58b420\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-sz5gdnfzqhdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\",\n      \"name\": \"2c72bb6d-878b-457e-bf39-56f98851abda\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-09T06:39:07.400427011Z\",\n      \"endTime\": \"2026-10-09T07:08:09.774067703Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5e53f64c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-p8759llzs4pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n      \"name\": \"ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:18:57.884545997Z\",\n      \"endTime\": \"2026-10-08T01:47:57.229221309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-n5sb8-kxqvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-n5sb8-kxqvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c5e53f64c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-p8759llzs4pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n      \"name\": \"ddc299f6-13e4-4aac-bd41-1e700b04d5a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:18:57.884545997Z\",\n      \"endTime\": \"2026-10-08T01:47:57.229221309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-n5sb8-kxqvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-n5sb8-kxqvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-p8759llzs4pn--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-n5sb8-kxqvs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25efd83de780\u003e: \n    failed to create NodePool np-drrgfm: failed waiting for nodepool=\"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\" in resourcegroup=\"rg-candidate-5-1-7sfzz5wmdjlq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\",\n      \"name\": \"c439fc17-ab56-478e-a949-fbf669f84a7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:47:33.608428033Z\",\n      \"endTime\": \"2026-10-08T02:16:36.341701384Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25efd83de780\u003e: \n    failed to create NodePool np-drrgfm: failed waiting for nodepool=\"np-drrgfm\" for cluster \"cluster-candidate-5-1-drrgfm\" in resourcegroup=\"rg-candidate-5-1-7sfzz5wmdjlq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c439fc17-ab56-478e-a949-fbf669f84a7b\",\n      \"name\": \"c439fc17-ab56-478e-a949-fbf669f84a7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:47:33.608428033Z\",\n      \"endTime\": \"2026-10-08T02:16:36.341701384Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-nhhtq to exist\\nMachine j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-7sfzz5wmdjlq-candidate-drrgfm--managed/providers/Microsoft.Compute/virtualMachines/j1s3k9h1b7u5q9w-np-drrgfm-hkbqc-x9l5r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2108427018348531712","occurred_at":"2026-10-09T05:19:08Z","signature_id":"688238fa4fec4ee5a50f4498249bef5708c8fb182d2ba3d5ee6a9bc11ef48c84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"0f183d4bcd814663af627cb09ff2b4f81f9b8d7b96a1ed9c5dbfef43215375ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"121af162046d8a171b60412751be255110229f88cfefd3ba18d5c6c164a8ae5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"14e87a585f2e303b894b77d8c959ae34784e5c6157129d38e9b5301844bb7ba0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"2f5ae43bb07be2044a3396af722d16a0065e608923ead5bdafb775bb39e37f4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"31263bdaf37c31d8b00994a69910ef2e640144e0087ea9bca2ce2644230f566e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"5e6c253faadc9bb7037e12aa7cd2177efa6e4d786e05f9ec021dc0d6ca95d93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"88e397f360afd6197fec72b913472ab67b4b2d10029ecc02621fb17a674f3316","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"9cbc0a353d999f7daabf373bfd9ff5feac8ee9d23b5b5e47d914749b0e6d8820","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"e5f07cf3f27270f92b263fd5d0ca25a5614664732ecf3e2c83484f30809f1bf7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0d3977ea4a90af04720c42d0bb759a3fb96691ce43398b60e6eef15c0a367212","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"35c1bfad26ec391bd6694018ed5a8d59c0813fc87ac12982198a869b9604acb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"68cb7350eeb61710d23cb67024cbbf9383969e0663d83204c8bdbdff0d33b967","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"698107089cf97b784786a6213e09a0a68dc2d0bd01e69c8e0e0c0fdb8f948aab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"8ee2deee6d590c8c6f98a003b663ac08a67a9a1ecc4cc6560bd842c5b1c9a52d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"a0dba5d1188bd276353fe4d2637a368162095ec26a8c24aee58b4ef68e6b2355","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"d3f83bccde47a3e6aa4acd1828dd9099b38db2477a89ab6bee0285d81eb16b5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"01163b61c17b562a500c04958cb33fe485d2c7ca115bc405d91028b41fee31fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"30c2dae7bbb702188aef5ab4608ef49e36dd251396ec6996c25febee6fad61be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"62b9a0dbb94dec0c48d4230d8c8526f7eb93e53eaeef32b48228dd8d42af77f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"6e47728030dcf1f6d034b909c5fda78d2f12b066fb41e2f513f61773969a2a38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"70ad7e4081cf1f599d09c2f83be92ab81460d5e3baf40247b0aaf860738f9066","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"bd4230f06a85eb9e1292a6291f9776d671fb62ec21c099d1d8dbdbd83275c152","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"cf1becd17e665f40e312fb2fb9948d50c9c5fa62d4421b6733acabe9bde7f248","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"ede7e9813082bc8cecccb538add5f1a3c53fe9bae98639f390f3a1e3e9e834e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"0295e7035dfce1af760e5a9f96d8886860036c81cb7a349f83dfe84f97f0ab25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"2efd6126d606d5f6f80d0316433b168030b43d3ea235668b13059c9b830d3aa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"660518fdfbb9f0800a819d55d8bc3171022d1eadfaf9a16981cf1f090bdaf04c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"844a548f21a121190f4b76ba3514e970ff80d7128375752094c4e416f2fd6f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"88363a2b12377ced6544609973aca118fa1d1df45bf3c3b0605e39c27653faec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"8a41ca55dc9a29f3bb19bad80e28580eee3b1db49cbcab92f2d225a074105caf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"a131321bd10141e15cfba687cd99d26f8cc540813c1b560df836c2ddf028dca9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"c1fc956072a43b79a620c5e7511a401ac3079dfd2ffe4780e4315cf518f89d75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"ffa11b5d4792edceb448fa6645e938d4beba1a8400ac52ce970668a6ebf44a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"42d95229988ca74bd7cb337361282d8c16d787466292deaf2232070c3cef245e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"636c0e8426e9e41af610392dfe2585499dc4ca5ac63cc2c2ad03af827766f74a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"6afa453b179354544c5915baf6f73ad97a3103ed41f793200617265657041048","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"ace0a1418b7953740f38672a49af405998a9e3e24c4a33b95e0d999dd68228b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"b2b0c546047cd7f8dfbc1e8fd084cbf9b0c93bccae10e310d13b26a94be9f023","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"d41cf6f362daae9f0f271e170de364045dff8bca3bf030691011bad678666f7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"def278b028e2cfc71409628aa852a649d7df162da0e1ca879d80fffe9d9e1dd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"f6435cbef4dc6ebe4912bc5681a94ef53c08760658835c2e2be55f09f269435c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"0af96c9de788a43bfc36cc28d4ef2f7ed16e382ed3dce340c647a636af5ecb10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"2d1db9807c28b66174164739cafde112b3198801994efbfaf7eb91ee2bc663c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"461d3cd85376afaa0deb74d80a0b8c0bf44e6e13b152912c7fa7055c0a01ba52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4acdd568139e3c3c134edd0d6f256da22e4a7a8368cc84137523b794cc84f751","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"87cadb376d6cd5fe3148fe99a8a0a312ae518ce1cd8d34338748d862f979c65e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"90e197c419ca4c3d130ecefd2a74215194ffd5df0642d2dc2d4c34686fc824f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9264fae13d8a0e126024fe87016ca0eb955eeacc351b1d83a0e62b7b2d4f8888","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e3312dc69a51a671f116346996b6b0e62dfdfde1bc0d102af6a4d41fef2b5576","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":29,"runs_affected":6,"failed_runs":6,"run_impact_percent":30,"anchor_occurrences":29,"after_last_push_count":71,"also_in":["PROD"],"trend_counts":[5,5,5,5,5,5,4,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x26308605f950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vb66qj77mjrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vb66qj77mjrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: cc37dc3c-22b2-4fec-a5d0-2aba23ea3ad5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x26308605f950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vb66qj77mjrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vb66qj77mjrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n            correlation ID: cc37dc3c-22b2-4fec-a5d0-2aba23ea3ad5\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xb11fa465920\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vsn9q285vfrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vsn9q285vfrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 264d5240-5762-4b86-9192-64fd260f49fd\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xb11fa465920\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vsn9q285vfrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vsn9q285vfrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n            correlation ID: 264d5240-5762-4b86-9192-64fd260f49fd\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3ab272e5a090\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-97t8mc9zr7vt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-97t8mc9zr7vt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: b25da80c-bc0c-47f5-b073-ca14b3b999a7\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3ab272e5a090\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-97t8mc9zr7vt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-97t8mc9zr7vt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n            correlation ID: b25da80c-bc0c-47f5-b073-ca14b3b999a7\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"2e2ec56fa735e3c6fb65ee60df50e7e7dace6166364141c3f1bfe0771ff4e918","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"344ee26cca90a92005b8b763492dab51ac06c3fcc9e83e7f0eae01cb2e7c5117","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"be78c1649579da41709f8c31cbe7770f3c078a5835b2696b6e43cfb6b4a7b282","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"ea927c68dc1333367697e506d3d20b5dcec61691018b2f2c2860e0c4025a33af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"1ddd75a4df7db175072426fb3e32a0f46c38e701f3c10c744a66995a09746157","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"2e8e7d85409aa17236b72b19ba8a8fe1c5ab5162a21ecac980d27db3bce6fd7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"65ac56874a943c9487d98905ee69c4459ed863f90e311c5e9ddd55258d7f6cf6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"6e040c991069a028c97b26eda22de505e4ce221c866b4b12509ad5f2670e7dc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"cce4a16ca669e9b7057b8665b57073714f79b4a75f6b8a8dd1bd6abaf244a6cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109056606967697408","occurred_at":"2026-10-10T23:00:54Z","signature_id":"047f3cf664fbe4c9399e7d9e39ace65b7b5034c798cabbf12a20ef71bdd4ce1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109056606967697408","occurred_at":"2026-10-10T23:00:54Z","signature_id":"0d1b7a19e089fa2f49fec182954f568d22b589137caccfa22b3310e2c248b52e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109056606967697408","occurred_at":"2026-10-10T23:00:54Z","signature_id":"2f15c844cb3bf2e547f3192ea8255b1fc50af63a7c115da87e8f9f189b3c8119","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109056606967697408","occurred_at":"2026-10-10T23:00:54Z","signature_id":"74b0bd407a7bf4288f79442643dffdc16dc3418593257db5b2ac62bb9b237bec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109056606967697408","occurred_at":"2026-10-10T23:00:54Z","signature_id":"9d743762d75d105744849f86d33b73961a8cbd62790ed58ecbd1d7f1ab753675","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","signature_id":"3c386237da479ec8664fa6d9ac7a9bd2aa5df117f4040d153a6a01fd3df4918e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","signature_id":"4eca13a165c2ccc73b835fae934265c145306a5b3c4d9ed3f076311aed85c4b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","signature_id":"500fd5ba3e339ff6a02a8c3df741232041d04709e81cedd8b10bf9374ceb0e4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","signature_id":"d81d7f06c05c906562011f64261d02893fd0c274bf91ed65c6010a47e11dd46c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","signature_id":"daeed2ecbf508b35f1ba38005d11ff1479283bbc484b275758e91ca9f68a2891","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","signature_id":"4755271a2405ebabdc1040cbbd3db7c2ac9758569fd67a3df2b4609761755cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","signature_id":"6a786858a0ce9d8898d3bee0c6e0bb2b32e2b8290a00cd0d692d1083e2a1c7db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","signature_id":"b206c5ae2b103cbfe005db15a1c02fa20fa420980cde7e1618e434e35e0da46e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","signature_id":"d6925f99dc273a7ab082759d35af63245cb77400725a1a98efeefc4b973a6b03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","signature_id":"f88d47a77e4c59b900e74f056ed5c5b9d70d2185cfe256eba9c161d0c2e7ae93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0660a8b07b67f6039cafbd2db5de6076302394cf9033181aa61d0e1bed619a61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"24834a656a673fd3cfb5ae1fccf25d73d8b576895e93c65a3c6a94a528913c38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"748728f92e96c01a73c8dc01210faf4d617f86fb186b83b70b3031c704b1a940","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"79bf72a228ccd77d5a3ed3ea923d269a222d78760e0d3456d70db4a2a4ed54ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"abb5a95e8ea83117472e6c379d204e8f2c25f4bdb19bf5b0daa764dda033af3b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":9,"runs_affected":6,"failed_runs":6,"run_impact_percent":30,"anchor_occurrences":9,"after_last_push_count":10,"also_in":["DEV","PROD"],"trend_counts":[0,8,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-08T00:03:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x273477d04048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-zvv7lmwhhzgh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n      \"name\": \"81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:30:01.277978624Z\",\n      \"endTime\": \"2026-10-08T01:59:06.873419943Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x273477d04048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-zvv7lmwhhzgh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n      \"name\": \"81d0dfd2-7968-4d7a-9a69-0ffb6aa5ea78\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:30:01.277978624Z\",\n      \"endTime\": \"2026-10-08T01:59:06.873419943Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2zm5h-ntjmk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-2zm5h-jnxj4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zvv7lmwhhzgh--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2zm5h-jnxj4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26a27898a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9xrgvjx8lwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"name\": \"e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:33.092346932Z\",\n      \"endTime\": \"2026-10-07T23:46:38.337633525Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26a27898a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9xrgvjx8lwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"name\": \"e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:33.092346932Z\",\n      \"endTime\": \"2026-10-07T23:46:38.337633525Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x390344d20c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4wvdmxwppl4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\",\n      \"name\": \"02042050-d676-4d90-bee1-de9c86994c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:29.815037524Z\",\n      \"endTime\": \"2026-10-07T14:59:36.125232922Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x390344d20c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4wvdmxwppl4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\",\n      \"name\": \"02042050-d676-4d90-bee1-de9c86994c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:29.815037524Z\",\n      \"endTime\": \"2026-10-07T14:59:36.125232922Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"ae1a8ec85d5f87964e54e270ab9ebf79bf92e62da6f496c93f46a3a9dfd56020","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6ba651d962b63ca9737c83935c51fabd9671a4349db5bc958c0be99ef420497d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"38e91e14bbfa3b78d5e999c68fea443396976fc0d1daad98d4cf67b9eaa12e1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"f9b070f99215534333fb93adb09f749de178f54c1544c512656282297ceb7a23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"f9af646ef53adca9c38d8c7a82da5f67f58665aa87755f378eaeda7b1930de7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"255e1dca1572a70b8f0d0f5f6aa345989e66a6b4f6f9e32911b805b3bfa7a379","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"c77d4d78f4573df1f5f58548e67fdb600dd43ca5bb56e96677224027c975bc44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"06edfc93dd116cf1f2412df4933dab83c6d60f345bd6545be6187369b744fbb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9174db2c962c5b14b002b9d9fc7ad502cc024cdfc29401bc9447ace098a0ae20","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bxmr827d8pqp\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":35,"anchor_occurrences":7,"after_last_push_count":7,"also_in":["DEV","PROD"],"trend_counts":[0,5,1,0,0,1,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":26,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1003aae08080\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n      \"name\": \"3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T23:48:35.970441715Z\",\n      \"endTime\": \"2026-10-12T00:17:41.715765376Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n            correlation ID: e2102b28-bcd5-415f-bde1-1dd34ba167b4\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1003aae08080\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dvn947jdn4hc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n      \"name\": \"3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T23:48:35.970441715Z\",\n      \"endTime\": \"2026-10-12T00:17:41.715765376Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dvn947jdn4hc--managed/providers/Microsoft.Compute/virtualMachines/q7p0f9j6y7m4m7c-np-deletable-glcrn-8z7zf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c051cc9-3f87-43a3-86ce-d9c6d0efccb9\n            correlation ID: e2102b28-bcd5-415f-bde1-1dd34ba167b4\n            client request ID: unavailable","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xcfd18f3b720\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r4vhjgdwsnjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n      \"name\": \"d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:22:41.727850228Z\",\n      \"endTime\": \"2026-10-08T02:51:47.319042789Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xcfd18f3b720\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r4vhjgdwsnjg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n      \"name\": \"d6c35801-94a0-4564-b906-f06f1cdcb28b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:22:41.727850228Z\",\n      \"endTime\": \"2026-10-08T02:51:47.319042789Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-b5742 to exist\\nMachine r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r4vhjgdwsnjg--managed/providers/Microsoft.Compute/virtualMachines/r5d8b8h3a4q7e4f-np-delete-main-fv67r-mqqrr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x115d2dac76c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-knjdfr9bnmx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"name\": \"3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:45:51.426276166Z\",\n      \"endTime\": \"2026-10-08T00:14:52.690397112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x115d2dac76c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-knjdfr9bnmx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"name\": \"3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:45:51.426276166Z\",\n      \"endTime\": \"2026-10-08T00:14:52.690397112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"a72db00a2257deae0a9b15a0618c4edf13182ab5933eabd4872272d30cdc4b95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"a84ae99184a527fede0f7b6d9bc32e2d01c9bea2f40994bc6848ccd2c629f84b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"41911d09cb3faeb31e1614a3ce95b71553ee40f58fbdbb9b2b3538c30fc780cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"e1898cdee251c0de6f399c876a928fe1ac01341d45cd38e4d6b00e9b3fb43608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"fbe36cbbceb1d0e1025eb3b54fd10fe074a1d442f305fcf1dd1cb97792c28108","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"c701b9c1b5bb023dff9bd4abb1de8fbc2c5ce5288ba7fb66eaf94d47e43c2e87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"6cb55ad6e469d73beffb431fa7679fc0430ca42fd9f6b8f1a796395bc2d38f01","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":10,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,2,5,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-mjltzvcvvmbh/cluster-nightly-4-22-xv7w42 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe85a0514ac0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xv7w42: failed waiting for cluster=\"cluster-nightly-4-22-xv7w42\" in resourcegroup=\"rg-nightly-4-22-mjltzvcvvmbh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n      \"name\": \"df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:26:58.898405913Z\",\n      \"endTime\": \"2026-10-12T23:56:08.010677982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n            correlation ID: 99498835-8536-458a-9168-4524819cc6bd\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-mjltzvcvvmbh/cluster-nightly-4-22-xv7w42 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe85a0514ac0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xv7w42: failed waiting for cluster=\"cluster-nightly-4-22-xv7w42\" in resourcegroup=\"rg-nightly-4-22-mjltzvcvvmbh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n      \"name\": \"df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:26:58.898405913Z\",\n      \"endTime\": \"2026-10-12T23:56:08.010677982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n            correlation ID: 99498835-8536-458a-9168-4524819cc6bd\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-lmbgqw\" at install version 4.22.0-0.nightly-multi-2026-10-12-042614\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2912ce16d00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-lmbgqw: failed waiting for cluster=\"cp-ystream-upgrade-4-23-lmbgqw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xm2k5t9rnhbb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n      \"name\": \"40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:30:45.035752487Z\",\n      \"endTime\": \"2026-10-12T23:59:45.878026173Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n            correlation ID: 61a61582-38bf-4dc1-a572-663613c43a27\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-lmbgqw\" at install version 4.22.0-0.nightly-multi-2026-10-12-042614\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2912ce16d00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-lmbgqw: failed waiting for cluster=\"cp-ystream-upgrade-4-23-lmbgqw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xm2k5t9rnhbb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n      \"name\": \"40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:30:45.035752487Z\",\n      \"endTime\": \"2026-10-12T23:59:45.878026173Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n            correlation ID: 61a61582-38bf-4dc1-a572-663613c43a27\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-gkft6x\" at install version 4.20.0-0.nightly-multi-2026-10-11-194900\nUnexpected error:\n    \u003c*fmt.wrapError | 0x168e93083220\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-gkft6x: failed waiting for cluster=\"cp-ystream-upgrade-4-21-gkft6x\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-p9dzhljjxbzb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87c1b701-975b-48a5-946a-b69c35657d88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87c1b701-975b-48a5-946a-b69c35657d88\",\n      \"name\": \"87c1b701-975b-48a5-946a-b69c35657d88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:28:15.763925029Z\",\n      \"endTime\": \"2026-10-12T23:57:15.727344969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87c1b701-975b-48a5-946a-b69c35657d88\n            correlation ID: 15d5486b-2716-4d22-b4be-42c25e5289af\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-gkft6x\" at install version 4.20.0-0.nightly-multi-2026-10-11-194900\nUnexpected error:\n    \u003c*fmt.wrapError | 0x168e93083220\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-gkft6x: failed waiting for cluster=\"cp-ystream-upgrade-4-21-gkft6x\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-p9dzhljjxbzb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87c1b701-975b-48a5-946a-b69c35657d88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87c1b701-975b-48a5-946a-b69c35657d88\",\n      \"name\": \"87c1b701-975b-48a5-946a-b69c35657d88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:28:15.763925029Z\",\n      \"endTime\": \"2026-10-12T23:57:15.727344969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87c1b701-975b-48a5-946a-b69c35657d88\n            correlation ID: 15d5486b-2716-4d22-b4be-42c25e5289af\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"32cce48c70cd78aaab0ca1cf29578d7e7f790aac0f49595cdef30c5b9fbbae37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"5b3bd45c8acbe1b8c11b2ffcb06fab0944c8108dcebe086fa39dfd4f4016f6f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"93d83589a75ababb8b2f24abad716697f692af996c7f0f1a8f51065a65539f5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"b867162c92e79c7b8c388045402cfc0fe873015bb7e68fb7d92ed60c1269d3aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"c15fdeb0bd481599ce3a3469d527a846870c693fc60106f4ccf60e4d23ab0f56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"6d2bb094474926c3944a6419013f592fd6ee220fa057099f1207616936341d71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"e4f5dc1ebfa32972bb080c0f1650a448a932216fc94a43ef0b38493ce1cd05c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"433ac3e41264da043a734cacbf5bc6523c2571e994c16b98d54f5b82c0cc686a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":10,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,1,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-23-jglxt7lcshss/cluster-nightly-4-23-r26z2q should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d73d9d10e00\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-r26z2q: failed waiting for cluster=\"cluster-nightly-4-23-r26z2q\" in resourcegroup=\"rg-nightly-4-23-jglxt7lcshss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\",\n      \"name\": \"41bf70ba-7344-4c62-9364-b6165697473d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:27:02.793421826Z\",\n      \"endTime\": \"2026-10-12T23:56:08.063137348Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-node-tuning-operator, dns-operator, machine-approver, cluster-storage-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-version-operator, ingress-operator, control-plane-pki-operator, olm-operator, cluster-network-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n            correlation ID: bbc14ae0-9ae0-4116-9b30-00ddeee9cda8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-23-jglxt7lcshss/cluster-nightly-4-23-r26z2q should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d73d9d10e00\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-r26z2q: failed waiting for cluster=\"cluster-nightly-4-23-r26z2q\" in resourcegroup=\"rg-nightly-4-23-jglxt7lcshss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\",\n      \"name\": \"41bf70ba-7344-4c62-9364-b6165697473d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:27:02.793421826Z\",\n      \"endTime\": \"2026-10-12T23:56:08.063137348Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-node-tuning-operator, dns-operator, machine-approver, cluster-storage-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-version-operator, ingress-operator, control-plane-pki-operator, olm-operator, cluster-network-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n            correlation ID: bbc14ae0-9ae0-4116-9b30-00ddeee9cda8\n            client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-phmgklt4jbx8/cluster-nightly-5-0-dsvpsx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x174e773b4b40\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-dsvpsx: failed waiting for cluster=\"cluster-nightly-5-0-dsvpsx\" in resourcegroup=\"rg-nightly-5-0-phmgklt4jbx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\",\n      \"name\": \"1dcda58e-002a-4f82-9591-34d179e38dd7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T23:24:10.263042602Z\",\n      \"endTime\": \"2026-10-11T23:53:11.680283328Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, azure-cloud-controller-manager, catalog-operator, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, kube-controller-manager, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, konnectivity-agent, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n            correlation ID: 8da2e5e4-c1bb-4598-8408-0edb1c690ef8\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-phmgklt4jbx8/cluster-nightly-5-0-dsvpsx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x174e773b4b40\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-dsvpsx: failed waiting for cluster=\"cluster-nightly-5-0-dsvpsx\" in resourcegroup=\"rg-nightly-5-0-phmgklt4jbx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\",\n      \"name\": \"1dcda58e-002a-4f82-9591-34d179e38dd7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T23:24:10.263042602Z\",\n      \"endTime\": \"2026-10-11T23:53:11.680283328Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, azure-cloud-controller-manager, catalog-operator, openshift-route-controller-manager, hosted-cluster-config-operator, olm-operator, openshift-controller-manager, cluster-policy-controller, kube-controller-manager, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, konnectivity-agent, cluster-image-registry-operator, cluster-version-operator, ingress-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1dcda58e-002a-4f82-9591-34d179e38dd7\n            correlation ID: 8da2e5e4-c1bb-4598-8408-0edb1c690ef8\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"e1a8bd9667a06efef689c609b6312d7ffbd0831f396ff2202ea50c17e42be65b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"994cc87ca03dcd70cb204d8d4904359ec0b786075af157e6024413d9b295804d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a3c74e7b398e04fb715ace48f34fd905bc9ad59fa3d32a732969fe4992fdd590","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, konnectivity-agent, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-4c2bdskg9txb/cluster-nightly-5-0-44gd6d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc4ca110a7e0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-44gd6d: failed waiting for cluster=\"cluster-nightly-5-0-44gd6d\" in resourcegroup=\"rg-nightly-5-0-4c2bdskg9txb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n      \"name\": \"bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:27:12.849511271Z\",\n      \"endTime\": \"2026-10-12T23:56:15.713757928Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, ingress-operator, konnectivity-agent, cluster-version-operator, cluster-storage-operator, community-operators-catalog, olm-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n            correlation ID: 37b4d83d-eab7-42e6-8d94-8b862a090192\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-4c2bdskg9txb/cluster-nightly-5-0-44gd6d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc4ca110a7e0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-44gd6d: failed waiting for cluster=\"cluster-nightly-5-0-44gd6d\" in resourcegroup=\"rg-nightly-5-0-4c2bdskg9txb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n      \"name\": \"bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:27:12.849511271Z\",\n      \"endTime\": \"2026-10-12T23:56:15.713757928Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, ingress-operator, konnectivity-agent, cluster-version-operator, cluster-storage-operator, community-operators-catalog, olm-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n            correlation ID: 37b4d83d-eab7-42e6-8d94-8b862a090192\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"99165d131f611e1774b5c8402a98875009d3632ad9f1c583245d77f5a5a09a7d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"cbedd273ed9a5b32964ea691b7c8952675d5b75c9ecaa831abc108bc9530aa44","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-11T23:00:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-nb9jxpzlfmcp/cluster-nightly-5-1-wkbflh should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d0dfed5cd40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-wkbflh: failed waiting for cluster=\"cluster-nightly-5-1-wkbflh\" in resourcegroup=\"rg-nightly-5-1-nb9jxpzlfmcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n      \"name\": \"e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T23:24:11.419225738Z\",\n      \"endTime\": \"2026-10-11T23:53:12.203667325Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-181926 is Partial (want Completed), started 22m49s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n            correlation ID: b50e9c9e-d4f4-4ba1-9e2c-863be02b98b4\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-nb9jxpzlfmcp/cluster-nightly-5-1-wkbflh should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d0dfed5cd40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-wkbflh: failed waiting for cluster=\"cluster-nightly-5-1-wkbflh\" in resourcegroup=\"rg-nightly-5-1-nb9jxpzlfmcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n      \"name\": \"e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-11T23:24:11.419225738Z\",\n      \"endTime\": \"2026-10-11T23:53:12.203667325Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-181926 is Partial (want Completed), started 22m49s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4b00e07-8ca7-4a34-a5ca-40ac469ad96a\n            correlation ID: b50e9c9e-d4f4-4ba1-9e2c-863be02b98b4\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109418788515483648","occurred_at":"2026-10-11T23:00:04Z","signature_id":"bcf7bc5e7d129d2651e6f43b84038ce2267b5bf6cb489b6ebbbdd577ed510c1b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"38696c55a4f3351eeef461deb7069c172bd5ca5ddee5655459be99523f263138","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x371cbc791620\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-n9bmt8cwvq6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\",\n      \"name\": \"7da88189-d1ef-4299-b869-95f7299be069\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:47:30.718672913Z\",\n      \"endTime\": \"2026-10-13T00:16:35.706367535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n            correlation ID: f89db514-fba8-4f47-b43b-0ea8cd9a0b5e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0x371cbc791620\u003e: \n    failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-n9bmt8cwvq6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\",\n      \"name\": \"7da88189-d1ef-4299-b869-95f7299be069\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-12T23:47:30.718672913Z\",\n      \"endTime\": \"2026-10-13T00:16:35.706367535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n            correlation ID: f89db514-fba8-4f47-b43b-0ea8cd9a0b5e\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"69b2e09a428ec7ca6e59b6def93f3906763678b69f1bb0465068db0038f587d3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5d4aaaaae5c748327f53adcfa681b6d36f3f8da08e1b106abec14dbcf1f183c1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down cluster resources, managed cluster, hosted cluster, namespaces; [descendantResources] remaining resources: applydesires, externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T05:17:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27901568a690\u003e: \n    resource group \"external-auth-cluster-ch86qj77dnn4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ch86qj77dnn4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"name\": \"0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T06:22:44.810261014Z\",\n      \"endTime\": \"2026-10-07T06:46:52.177879454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x279015820520\u003e{\n                msg: \"resource group \\\"external-auth-cluster-ch86qj77dnn4\\\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0x279015820460\u003e{\n                    msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*fmt.wrapError | 0x279015820440\u003e{\n                        msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                        err: \u003c*fmt.wrapError | 0x279015820420\u003e{\n                            msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            err: \u003c*exported.ResponseError | 0x27901606a960\u003e{\n                                ErrorCode: \"InternalServerError\",\n                                StatusCode: 200,\n                                RawResponse: {\n                                    Status: \"200 OK\",\n                                    StatusCode: 200,\n                                    Proto: \"HTTP/2.0\",\n                                    ProtoMajor: 2,\n                                    ProtoMinor: 0,\n                                    Header: {\n                                        \"Cache-Control\": [...],\n                                        \"X-Envoy-Upstream-Service-Time\": [...],\n                                        \"X-Ms-Operation-Identifier\": [...],\n                                        \"X-Ms-Correlation-Request-Id\": [...],\n                                        \"X-Msedge-Ref\": [...],\n                                        \"Pragma\": [...],\n                                        \"Content-Type\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [...],\n                                        \"Strict-Transport-Security\": [...],\n                                        \"X-Content-Type-Options\": [...],\n                                        \"Date\": [...],\n                                        \"Content-Length\": [...],\n                                        \"X-Ms-Request-Id\": [...],\n                                        \"X-Ms-Routing-Request-Id\": [...],\n                                        \"X-Cache\": [...],\n                                        \"Expires\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [...],\n                                    },\n                                    Body: \u003c*exported.nopClosingBytesReader | 0x279015820220\u003e{\n                                        s: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        i: 0,\n                                    },\n                                    ContentLength: 1273,\n                                    TransferEncoding: nil,\n                                    Close: false,\n                                    Uncompressed: false,\n                                    Trailer: nil,\n                                    Request: {\n                                        Method: \"GET\",\n                                        URL: {Scheme: ..., Opaque: ..., User: ..., Host: ..., Path: ..., Fragment: ..., RawQuery: ..., RawPath: ..., RawFragment: ..., ForceQuery: ..., OmitHost: ...},\n                                        Proto: \"HTTP/1.1\",\n                                        ProtoMajor: 1,\n                                        ProtoMinor: 1,\n                                        Header: {...: ..., ...: ...},\n                                        Body: nil,\n                                        GetBody: nil,\n                                        ContentLength: 0,\n                                        TransferEncoding: nil,\n                                        Close: false,\n                                        Host: \"management.azure.com\",\n                                        Form: nil,\n                                        PostForm: nil,\n                                        MultipartForm: nil,\n                                        Trailer: nil,\n                                        RemoteAddr: \"\",\n                                        RequestURI: \"\",\n                                        TLS: nil,\n                                        Cancel: nil,\n                                        Response: nil,\n                                        Pattern: \"\",\n                                        ctx: \u003csuppressed context\u003e,\n                                        pat: nil,\n                                        matches: nil,\n                                        otherValues: nil,\n                                    },\n                                    TLS: {\n                                        Version: 772,\n                                        HandshakeComplete: true,\n                                        DidResume: false,\n                                        CipherSuite: 4866,\n                                        CurveID: 25,\n                                        NegotiatedProtocol: \"h2\",\n                                        NegotiatedProtocolIsMutual: true,\n                                        ServerName: \"management.azure.com\",\n                                        PeerCertificates: [..., ..., ...],\n                                        VerifiedChains: [...],\n                                        SignedCertificateTimestamps: nil,\n                                        OCSPResponse: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        TLSUnique: nil,\n                                        ECHAccepted: false,\n                                        HelloRetryRequest: true,\n                                        ekm: 0x585e900,\n                                        testingOnlyPeerSignatureAlgorithm: 2052,\n                                    },\n                                },\n                                errMsg: \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27901568a690\u003e: \n    resource group \"external-auth-cluster-ch86qj77dnn4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ch86qj77dnn4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"name\": \"0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T06:22:44.810261014Z\",\n      \"endTime\": \"2026-10-07T06:46:52.177879454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x279015820520\u003e{\n                msg: \"resource group \\\"external-auth-cluster-ch86qj77dnn4\\\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0x279015820460\u003e{\n                    msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*fmt.wrapError | 0x279015820440\u003e{\n                        msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                        err: \u003c*fmt.wrapError | 0x279015820420\u003e{\n                            msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            err: \u003c*exported.ResponseError | 0x27901606a960\u003e{\n                                ErrorCode: \"InternalServerError\",\n                                StatusCode: 200,\n                                RawResponse: {\n                                    Status: \"200 OK\",\n                                    StatusCode: 200,\n                                    Proto: \"HTTP/2.0\",\n                                    ProtoMajor: 2,\n                                    ProtoMinor: 0,\n                                    Header: {\n                                        \"Cache-Control\": [...],\n                                        \"X-Envoy-Upstream-Service-Time\": [...],\n                                        \"X-Ms-Operation-Identifier\": [...],\n                                        \"X-Ms-Correlation-Request-Id\": [...],\n                                        \"X-Msedge-Ref\": [...],\n                                        \"Pragma\": [...],\n                                        \"Content-Type\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [...],\n                                        \"Strict-Transport-Security\": [...],\n                                        \"X-Content-Type-Options\": [...],\n                                        \"Date\": [...],\n                                        \"Content-Length\": [...],\n                                        \"X-Ms-Request-Id\": [...],\n                                        \"X-Ms-Routing-Request-Id\": [...],\n                                        \"X-Cache\": [...],\n                                        \"Expires\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [...],\n                                    },\n                                    Body: \u003c*exported.nopClosingBytesReader | 0x279015820220\u003e{\n                                        s: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        i: 0,\n                                    },\n                                    ContentLength: 1273,\n                                    TransferEncoding: nil,\n                                    Close: false,\n                                    Uncompressed: false,\n                                    Trailer: nil,\n                                    Request: {\n                                        Method: \"GET\",\n                                        URL: {Scheme: ..., Opaque: ..., User: ..., Host: ..., Path: ..., Fragment: ..., RawQuery: ..., RawPath: ..., RawFragment: ..., ForceQuery: ..., OmitHost: ...},\n                                        Proto: \"HTTP/1.1\",\n                                        ProtoMajor: 1,\n                                        ProtoMinor: 1,\n                                        Header: {...: ..., ...: ...},\n                                        Body: nil,\n                                        GetBody: nil,\n                                        ContentLength: 0,\n                                        TransferEncoding: nil,\n                                        Close: false,\n                                        Host: \"management.azure.com\",\n                                        Form: nil,\n                                        PostForm: nil,\n                                        MultipartForm: nil,\n                                        Trailer: nil,\n                                        RemoteAddr: \"\",\n                                        RequestURI: \"\",\n                                        TLS: nil,\n                                        Cancel: nil,\n                                        Response: nil,\n                                        Pattern: \"\",\n                                        ctx: \u003csuppressed context\u003e,\n                                        pat: nil,\n                                        matches: nil,\n                                        otherValues: nil,\n                                    },\n                                    TLS: {\n                                        Version: 772,\n                                        HandshakeComplete: true,\n                                        DidResume: false,\n                                        CipherSuite: 4866,\n                                        CurveID: 25,\n                                        NegotiatedProtocol: \"h2\",\n                                        NegotiatedProtocolIsMutual: true,\n                                        ServerName: \"management.azure.com\",\n                                        PeerCertificates: [..., ..., ...],\n                                        VerifiedChains: [...],\n                                        SignedCertificateTimestamps: nil,\n                                        OCSPResponse: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        TLSUnique: nil,\n                                        ECHAccepted: false,\n                                        HelloRetryRequest: true,\n                                        ekm: 0x585e900,\n                                        testingOnlyPeerSignatureAlgorithm: 2052,\n                                    },\n                                },\n                                errMsg: \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"bbd40a695906a4219c65ee7a3a59b31da7d9339ef823acf986d08e0c6c81eb03","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"e16b800ac434d36ffa1d42c492b02c9a13d0e3977b63166c8bc11fbd9845e06f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["PROD"],"trend_counts":[6,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-08T00:03:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x86f41c2a240\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n      \"name\": \"1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:18:56.823039116Z\",\n      \"endTime\": \"2026-10-08T01:42:56.76971285Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x86f41c2a240\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-p8d8rc9ts7fq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n      \"name\": \"1e2cfd55-dd4d-47e4-8c4b-c3e1f95ed759\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:18:56.823039116Z\",\n      \"endTime\": \"2026-10-08T01:42:56.76971285Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107985213920382976","occurred_at":"2026-10-08T00:03:34Z","signature_id":"bcab36d182af2adfa4121f84261759d0fa33d47d8fccb1037ef350c84a20b435","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96eb8165e1517c94f94344a079943853da38a18f87e198ac97e2ca5f84f1f2e3","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x3d800eb17200\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3d800f297260\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3d800f297240\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3d800e191e00\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3d800e204000\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3d800eb05ee0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3d800e0fab40\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x3d800eb17200\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3d800f297260\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3d800f297240\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3d800e191e00\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3d800e204000\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x3d800eb05ee0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x3d800e0fab40\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"bcf5d924fc188320a62950d409f473b61ec182e32796d6b849eb900353c06dc6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"eba9f0207784082ae028e0292575147a8480a2f1068a534a4622234669f79e31","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-08T23:00:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-20-g5bgdz\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2608a89861e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2608a90941e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2608a90941c0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2608a81192c0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2608a86f4180\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2608a86f4100\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2608a7f286e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-20-g5bgdz\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x2608a89861e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2608a90941e0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2608a90941c0\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2608a81192c0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2608a86f4180\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2608a86f4100\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2608a7f286e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108331766338031616","occurred_at":"2026-10-08T23:00:38Z","signature_id":"9d68ac0cb994edda68255e9dddbaf9ce4fa7933fd1c252de20b095bdb9081302","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"88cee2a035a0f46609d44b8db5480aa9ed0efd1c5fec7797c0cbce1364777b87","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-10-06..2026-10-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-07T13:37:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2655580ad6c0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2655580ad6c0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"28108545b283b909283d0a8f87e72879ae1a1db48c35cd0650ff7b4851a09bcb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}