{"meta":{"start_date":"2026-10-12","end_date":"2026-10-25","start_at":"2026-10-12T00:00:00Z","end_at":"2026-10-26T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T08:15:28Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":165,"failed_runs":121,"raw_occurrences":318,"matched_occurrences":310,"runs_affected":112},"rows":[{"environment":"dev","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":42,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":42,"after_last_push_count":1,"trend_counts":[42,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x212fbc084a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-8hkmlh8qnx7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"name\": \"a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:53.824928754Z\",\n \"endTime\": \"2026-10-12T16:34:00.204962099Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n correlation ID: 2d28ca59-634e-45e7-aa28-3231a3cd431f\n client request ID: 9ee94869-89d4-4850-9db2-977e7eaffbfa\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x212fbc084a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-8hkmlh8qnx7w\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"name\": \"a5d399ee-b335-46b2-ac26-ebca4137d374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:53.824928754Z\",\n \"endTime\": \"2026-10-12T16:34:00.204962099Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5d399ee-b335-46b2-ac26-ebca4137d374\n correlation ID: 2d28ca59-634e-45e7-aa28-3231a3cd431f\n client request ID: 9ee94869-89d4-4850-9db2-977e7eaffbfa\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d33bff38b20\u003e: \n failed to create HCP cluster ea-listhxx92t: failed waiting for cluster=\"ea-listhxx92t\" in resourcegroup=\"ea-list-6s5nhp2lwjz2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"name\": \"b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:25.793338411Z\",\n \"endTime\": \"2026-10-12T16:34:29.312060412Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n correlation ID: c3bdb4b5-d9e1-4722-8d4b-0801c32c52a6\n client request ID: 6f856d28-2856-4395-afc1-9135fdfb41b9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d33bff38b20\u003e: \n failed to create HCP cluster ea-listhxx92t: failed waiting for cluster=\"ea-listhxx92t\" in resourcegroup=\"ea-list-6s5nhp2lwjz2\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"name\": \"b3c17246-a7f7-45c8-8a03-331ddc91bb74\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:25.793338411Z\",\n \"endTime\": \"2026-10-12T16:34:29.312060412Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3c17246-a7f7-45c8-8a03-331ddc91bb74\n correlation ID: c3bdb4b5-d9e1-4722-8d4b-0801c32c52a6\n client request ID: 6f856d28-2856-4395-afc1-9135fdfb41b9\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3390eb576ac0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pj6qdmqzbvvk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\",\n \"name\": \"9ca81053-a842-4371-b45d-1725686e5689\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:50.549768464Z\",\n \"endTime\": \"2026-10-12T16:34:00.22620878Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n correlation ID: bd5b8009-857d-40ed-a78e-2ded6716c304\n client request ID: bd2fbaf8-a68b-4aff-9d2f-f237a37aad2f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3390eb576ac0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pj6qdmqzbvvk\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\",\n \"name\": \"9ca81053-a842-4371-b45d-1725686e5689\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:04:50.549768464Z\",\n \"endTime\": \"2026-10-12T16:34:00.22620878Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ca81053-a842-4371-b45d-1725686e5689\n correlation ID: bd5b8009-857d-40ed-a78e-2ded6716c304\n client request ID: bd2fbaf8-a68b-4aff-9d2f-f237a37aad2f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"055f9a9b6c244ec70223014863e02389b9a479621a6d56cd84fc1a590883325e","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"06d984b5bd61e16ead3a82aa893ca35dc9577f417c04093909481769e3f573b8","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"0a7b025a664cd937a27c1586ab0853b5c1ef3166363dadc1edebbd26f45f0f59","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"160071fa1d66581da10fab1353ab09d80ff5734975070cab9434ca0e1322ff49","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"1ec3f7dc73b39cc393755b265af0c51797463b27d90c21e0a2834ab16544dbf0","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"40e270b1250c572d95ad297676751fd03a52912e32aa17a878d7239f4fb1933c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"434f1f50bc3fec5a453eb5fc6ce070a5ad6cf40f73119f0ef622c86873aa56ea","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"44ded8bb1a8d8a2ac19d80e715a411a4b8efc7b7cc9f887538780c1e34b93ab9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"47dea9fa6d473ca88a0d6daa3a8a61ce6d04ebe38764f2f66d84f10a437111c6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"4d85fc8c6214af114002c843ebdc86a94cf3126e2e1355356d3ebd73640aa90f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"51071ac504e6050a1fd884cd82764171a1d38ab7411e9ae858f536e87567f055","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"586593e2fff2e69d645fc42dff24f41310743ceba1deccc3b657c96749c0928b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5bc4f6a77655afb07a6100dcfe611731a929b63429a3f3272baa0c1801222753","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5c7543311511f19fcd39fd25b976d88fc18cf2a06f89b8cd4da4fdcb65f57861","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"5f344ec756d50afef29819a6882bec64ddbd8b7b94b117d33badf95b4228e5c9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"61789678564be0aad54c9098dc019c63c0a07abf6db9351123669661958b90b1","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6b095616bded2736b44fd15719c1d4b68c8ddb854dc8491ee2f30e52888a0be7","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6e1b7db0d8aeab353439c24ad4514730ee14c5e2353bca21858fe1f9675242cc","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"7c6abafe393255f1739fdb387bd94e187d718daaaf8baf7ec69854b4ddf141b2","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"7cb425f2d7951985395b91345c374be4c0dca4d08831fcfb07f410b4baec6807","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"8181f041b308c072f1833b4eb56517c581e5c8e4e9cd0a8ea0e6808fccc7a57f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"844afb351725b6475ddd327ef78fbf5da706650b28665d4a6e378e60144b8e25","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"85acf0e937ffe0027b15bd44eca4c7c42194560ce1678ed387a13da6d7aadcb6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"945c8b8ba229efce78c82c55117d9808b09ecb3e3f5b4cd8cda0f81bc283543f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"97fe8f81d50ab0c04cc31936087a1cb6ccf4abd2ff0897e188700ec58b85890f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9b8bdf9e25347ed8bbd86e38e47f8af0335efec4efa7ea5222d2c6561d3fadfb","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9f01d929e13b36781c18a8d7f12e2b1fcb9913e0a3fcb25d7d5ff885c820a357","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"9f58aa2dc15f7e7336a495843cc0f699c3e5b143f75bf07f3842358d658bcdca","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"a0f1bf26985a99e6539af709b7c6cfaeed3b89c57f24089e9a344841e1966c77","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"a312b9a0ce9a609eef608f7ee363a1a717dc275fe1d0ca99a6b39591f5f7c10b","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b2b126547a0bcacd7f8d6fadea91d4a58f4981406c524e6014bd0b7da6c92d4d","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b7d3cff9604baa7adb7447f8feddb9b4e1f30933ea4005d1bf550006ccbe8684","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"b83badcc40496e121c06a8423ede3d7aeada8dd0924a4287300c02e0fa12d3e6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"c7ae04b38766210a8108f94bb04d957dfc5bca36ff0aa8754d485ec2530ed6a3","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d1805d4cf77820ef4f29d07237f0247a441e0228a2174ebcae5ceacb2de61aa9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d859241facd880cf6291230bc4ecdb749dce2456ddc8c4a192721ae181952d05","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"da295e3f891ebd53e53b834c83d5f2d2d8c7990e91838251fdcf5881a2a1f90a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"dbf86b79040a8aaf8dd94a9133ea7c86d12cf15986aa135f1c6a41485545be9c","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e642f26ea0a98605c1da7d415345036ed06aed0f4668bc7504474fbfa4f815a5","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e76fe38f94cc0c24f57a2b14af3523d40d7b5c15cd0e3fefac98dc8b85b99bed","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"ed62ccd125102b89fe1e6cd6a42a65c596b410194665770b4c1ecb63e1ef3a7a","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"ed6cd23384ef866a46342ced82dc3a93106a5447a643b0459d4e1595765f54a8","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":10.909090909090908,"anchor_occurrences":18,"after_last_push_count":19,"trend_counts":[5,13,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":42,"prior_last_seen_at":"2026-10-13T18:01:34Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":18}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T20:26:27Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1682688-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.46:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1682688/providers/microsoft.monitor/accounts/services-j1682688\", namespace=\"klusterlet-2te3djdf1hkgonc8k4i71prc94m2s3ap\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qlgkj\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te3djdf1hkgonc8k4i71prc94m2s3ap on cluster ci01-j1682688-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T19:14:04Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9671808-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.101:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9671808/providers/microsoft.monitor/accounts/services-j9671808\", namespace=\"klusterlet-2te2bel6rgtolfhffo3974figin61sko\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7lt7\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te2bel6rgtolfhffo3974figin61sko on cluster ci01-j9671808-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T19:29:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1296000-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1296000/providers/microsoft.monitor/accounts/services-j1296000\", namespace=\"klusterlet-2te2i26cvbt70m1icpl3fmd77s52ciq8\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-86rpv\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te2i26cvbt70m1icpl3fmd77s52ciq8 on cluster ci01-j1296000-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","occurred_at":"2026-10-13T18:01:34Z","signature_id":"dea3871c0f87e0576e988742c681e43a182dd667db806dd19107b71b0897c1c0","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110043835399671808","occurred_at":"2026-10-13T16:23:47Z","signature_id":"75b1290532c2ce606ea20bb999b66cb1911da401025d28a68dcef6b1a4ebb765","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039512951296000","occurred_at":"2026-10-13T16:06:36Z","signature_id":"c8d1bc3849a91c7150052cbd8fd01e8bacd6acfee15dec3a5b9f1ee2a8d69a1f","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039423042195456","occurred_at":"2026-10-13T16:05:54Z","signature_id":"e0aece362630a726f3a415cc6e772ac06ec61590d1f6668bb57f6d646bc54566","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","occurred_at":"2026-10-13T15:48:44Z","signature_id":"dc0f6ac01acdec662cd08a68e54dfaa04653844d8ecf2ea1d8d22da7ec3a472c","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","occurred_at":"2026-10-13T15:36:01Z","signature_id":"545a94509aea8d174b7c626867e4c3ee62e7a7063a644a5bd20cd3227ea583e7","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"d7646a570dc56c8aaf2ff17561f2018e209352457d4d221d6f98298c58002476","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","occurred_at":"2026-10-13T13:07:01Z","signature_id":"84707bded3f4fb605d70b06f7b06b6aaa5f660163675ed498765d69734732aa9","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109985456534851584","occurred_at":"2026-10-13T12:31:49Z","signature_id":"e0d8a21444c75df32d1ee0ca0a5977901884526027e6eba129d7da65137483ea","pr_number":7441,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"45a81790b69f5a13e41c3f6d58353b78d39c6df6f209d0a628107acd5871dc3b","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","occurred_at":"2026-10-13T11:41:15Z","signature_id":"c0807ee7633cb3b74a1bec10eeb1a189d4d2920a14c2e55e84330c489057c130","pr_number":7484,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109920002722238464","occurred_at":"2026-10-13T08:11:43Z","signature_id":"a19d81f743cded8d0f7583e6aed1c76e28428956863bd4963ceab29f6d4d2f68","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","signature_id":"4a20c7552b13dbe39fee7488dec78de85e3480b9fcb8c75ca75d30c3902dc95f","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754180636774400","occurred_at":"2026-10-12T21:12:48Z","signature_id":"54a7b1283931df6f61be82ca63d92bf372c766203c87971d1d688d38deb1c668","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"a67e3b1ce6606aadaaf58795784360bda0ae262c0ccd8be147cd4aa4d8cfa336","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109566058561015808","occurred_at":"2026-10-12T08:45:16Z","signature_id":"16da3c90424d3e20001b8e7c2d884bfc967a23a24761b0cf7cc40ac186996cdd","pr_number":7219,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7458/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528983438626816","occurred_at":"2026-10-12T06:17:57Z","signature_id":"9a0460e320537545b7df7ed4ea07d8af2e7eacb2bec27a23e38614a9b14396d9","pr_number":7458,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","signature_id":"7f1c703708a0c73e567c797b757f2be0ee02cc20a38341bda264d7aaee221d0e","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":17,"runs_affected":17,"failed_runs":17,"run_impact_percent":10.303030303030303,"anchor_occurrences":17,"after_last_push_count":64,"trend_counts":[8,9,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":204,"prior_last_seen_at":"2026-10-13T17:40:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":17}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T20:03:58Z\n Ended: 2026-10-13T20:15:58Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6775168-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:58:36Z\n Ended: 2026-10-13T19:18:32Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1536384-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:50:39Z\n Ended: 2026-10-13T16:56:39Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1238272-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034896461238272","occurred_at":"2026-10-13T15:47:07Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","occurred_at":"2026-10-13T15:36:01Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7499,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028695728033792","occurred_at":"2026-10-13T15:23:37Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028457319600128","occurred_at":"2026-10-13T15:22:41Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","occurred_at":"2026-10-13T11:41:15Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7484,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","pr_number":7491,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","signature_id":"ec53f16855846d8bcc63c32e61dda8320f5f786dcafa5b90527c9de5094abd44","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109622919087788032","occurred_at":"2026-10-12T12:31:13Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109612109212946432","occurred_at":"2026-10-12T11:48:16Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7467,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109555123024826368","occurred_at":"2026-10-12T08:00:48Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6746,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":15,"runs_affected":4,"failed_runs":4,"run_impact_percent":2.4242424242424243,"anchor_occurrences":15,"after_last_push_count":5,"trend_counts":[3,12,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":133,"prior_last_seen_at":"2026-10-13T15:48:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x31b87eaf00d8\u003e: \n resource group \"node-limit-test-2ch85v6j6t94\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2ch85v6j6t94\" to finish deleting: GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"name\": \"2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:18.781083053Z\",\n \"endTime\": \"2026-10-13T17:01:24.31591322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0s3j58fj1f3fm4vsgrm1lrp3it3l1 still exists (deletion dispatched at 2026-10-13T16:37:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n correlation ID: 879b6302-da33-4ea2-941e-e29845fe6259\n client request ID: f5a45ff4-173b-4f67-a8d6-7bf91987cdf6\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x31b87eaf00d8\u003e: \n resource group \"node-limit-test-2ch85v6j6t94\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2ch85v6j6t94\" to finish deleting: GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"name\": \"2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:18.781083053Z\",\n \"endTime\": \"2026-10-13T17:01:24.31591322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0s3j58fj1f3fm4vsgrm1lrp3it3l1 still exists (deletion dispatched at 2026-10-13T16:37:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n correlation ID: 879b6302-da33-4ea2-941e-e29845fe6259\n client request ID: f5a45ff4-173b-4f67-a8d6-7bf91987cdf6\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3bf37649c048\u003e: \n resource group \"customer-rg-kptnhtffptgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-kptnhtffptgl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"name\": \"d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:42:39.455530141Z\",\n \"endTime\": \"2026-10-13T17:06:41.723960509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0oaptqtlfm45pi2ru69verofh93nj still exists (deletion dispatched at 2026-10-13T16:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n correlation ID: 57954ad9-2ba3-4943-8c75-f5745f7ae1f7\n client request ID: bbb4049f-9eb5-4ee1-9bdd-ccda4efc41c1\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3bf37649c048\u003e: \n resource group \"customer-rg-kptnhtffptgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-kptnhtffptgl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"name\": \"d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:42:39.455530141Z\",\n \"endTime\": \"2026-10-13T17:06:41.723960509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0oaptqtlfm45pi2ru69verofh93nj still exists (deletion dispatched at 2026-10-13T16:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n correlation ID: 57954ad9-2ba3-4943-8c75-f5745f7ae1f7\n client request ID: bbb4049f-9eb5-4ee1-9bdd-ccda4efc41c1\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x309531b0198\u003e: \n resource group \"ea-list-wtljvtmmhcsk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list8h4wvd\" in resourcegroup=\"ea-list-wtljvtmmhcsk\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"name\": \"2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:18.149454561Z\",\n \"endTime\": \"2026-10-13T17:02:21.62230875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6snbp7v915nacod7cib95rmn96j still exists (deletion dispatched at 2026-10-13T16:38:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n correlation ID: 8d339200-f0e0-47b8-90d7-53a61a990c91\n client request ID: 742fa150-0903-4156-b9ea-5d97b505f86c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x309531b0198\u003e: \n resource group \"ea-list-wtljvtmmhcsk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list8h4wvd\" in resourcegroup=\"ea-list-wtljvtmmhcsk\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"name\": \"2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:18.149454561Z\",\n \"endTime\": \"2026-10-13T17:02:21.62230875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6snbp7v915nacod7cib95rmn96j still exists (deletion dispatched at 2026-10-13T16:38:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n correlation ID: 8d339200-f0e0-47b8-90d7-53a61a990c91\n client request ID: 742fa150-0903-4156-b9ea-5d97b505f86c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","occurred_at":"2026-10-13T15:48:44Z","signature_id":"ed688746e174fb91b628587af9f6d4fc2c44f7a9d83bdae64d4e89a50cd9e76a","pr_number":7489,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"022aadd1c5b43f5fcb46b49de5bea32873432564b44b704b42ed859546fe8e82","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"6d01e4638c3973f17302f07ed79508b57643e22d74bb892cac9c1af52e92a292","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"72c1ad55e4062aec518f3f62817b93e6395a0432a25afcb61cb220d43060d265","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"7be575f2f85e63d8ce2308f15291aa10c120d9bccad74cdd55d95042f97a55a5","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"94aae066f49ff828f8658c561f57e2a599bd9f4bcee2e2d506ad8011ab4f1278","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"a17143f3d84444b29b8010bee708abc0adf653cb05b16055bea351d2e9fb5d99","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"b92ba843e0e832a2ad56a0bbd96074f26f53e9816edef7f8e3b6ac20d8300aff","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"bf1c9ad85dc67c52bb5db81e418c326fc84cc55cd5304ca050cd7f99526fd13a","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"d91ceb0c461dd8cc9e4f63cf5a15d575daca06c95808c7ef0af47faf649a13d6","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"e5551334aa432c7f1d4fcaa4df0fbb0de936bcd598badb4046f07fb57beb7449","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"ecc5ff218b668fca411a6a387f8fa1e8cc8fcb2c2b7b038bd92d4f9b96814b90","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109659008674566144","occurred_at":"2026-10-12T14:54:37Z","signature_id":"aa1b645215452d614b0ab2fb286d0f4c2a52df3c3bf4b0684396494f99dc7271","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"9464cf8289c40c4ace5feeaa2af2e60f2ff16f8a6b1306bce054078519a3b607","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"c8fe87bb1fbea1d2b82daec5c2abcf5f379166a6c5ecbc60a09403cacb34657b","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":14,"runs_affected":14,"failed_runs":14,"run_impact_percent":8.484848484848484,"anchor_occurrences":14,"after_last_push_count":16,"trend_counts":[7,6,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":69,"prior_last_seen_at":"2026-10-14T04:11:31Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":14}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-14T05:54:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9250048-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9250048/providers/microsoft.monitor/accounts/hcps-j9250048\", namespace=\"ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ffckj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-14T05:54:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9250048-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.191:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9250048/providers/microsoft.monitor/accounts/hcps-j9250048\", namespace=\"ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ffckj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T23:32:29Z\n Ended: 2026-10-13T23:45:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T23:32:29Z\n Ended: 2026-10-13T23:45:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T22:15:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4787072-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4787072/providers/microsoft.monitor/accounts/hcps-j4787072\", namespace=\"ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vkmx4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T22:15:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4787072-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4787072/providers/microsoft.monitor/accounts/hcps-j4787072\", namespace=\"ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vkmx4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110221944149250048","occurred_at":"2026-10-14T04:11:31Z","signature_id":"7756294fda68fe98e6d3f7d273b6b789955b562628b9575f65001f0516fb17ea","pr_number":6712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"81e6aedf0da4f27efde8cb67872b6903101fe3182111cf67d2dfdc5646317ecb","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","signature_id":"4587179fd233fc65eb9fdb22388faacf6ec23475c73d8a87452ef9bb1dc63ec6","pr_number":7503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"1adc25e471ad62d434570973bdec6d11ffd2186cddd280c054d7dedcef620819","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"683b9a267c4b1dc9692126fc62980df123b055e51ab1e62c31bcb909366380a0","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","signature_id":"d802fbeb68fcabadbc0eb4bc2365b8c44020b5b0b251941d4bface05433d26f5","pr_number":7491,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","occurred_at":"2026-10-13T00:34:34Z","signature_id":"2782cf0dfc0a0c20b3659d17b6c3a4c2d2b8a0833ca169e4ccba3462287f86c1","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"5cdd689b990f511feccbe071c3661b0d836cd6b71dcc489745470445ffd67711","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109740763322519552","occurred_at":"2026-10-12T20:19:29Z","signature_id":"1f8e213972bab57fbe586fb0bcc701f04d1d69dc2413a26b541db4737b3a7545","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"ace1503ea85ab8f2b4df98ae24ce5d1ca17bf7dd773d4068dfe89225c5ed9594","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"18f5e818c6420e89e3d88cbc04dfa007473088c4fb38a457467fa320874574d7","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"a980a63c43707ea1bb98420a0dc8810e1d7f957a882e906bd2844388f2b4bbc5","pr_number":7468,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"c2c5cbd8dc969313ed2d41b08a27fef289acf32d3365fe9e6a45d7383b454d11","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109528755037802496","occurred_at":"2026-10-12T06:17:02Z","signature_id":"36fc6cf9b7b57226fae3978e4f0d34c96bf304895ef33b2f585810930ebddbca","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-2dtwt4gvf9v7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":13,"runs_affected":13,"failed_runs":13,"run_impact_percent":7.878787878787879,"anchor_occurrences":13,"after_last_push_count":18,"trend_counts":[5,6,2,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":40,"prior_last_seen_at":"2026-10-14T03:39:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":13}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-pljzvnf827th/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-pljzvnf827th/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-jf2trk9s8chj/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-jf2trk9s8chj/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-wvbxxwdg6hgk/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-wvbxxwdg6hgk/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213970672488448","occurred_at":"2026-10-14T03:39:51Z","signature_id":"fc73da3f64df28e28a5f3d03ec006882753ad077ad7fd997dd91714a48b160a9","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"aba6bac3ecf6fccefb4d5d45429681cbe92d317984bf96d8fe7f780c1e7bca9a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110137118025060352","occurred_at":"2026-10-13T22:34:27Z","signature_id":"94d1e13589455fbe3faae0d56a4527d0080bf018f5cc0df414cf6c40b010f2da","pr_number":6712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","occurred_at":"2026-10-13T21:41:52Z","signature_id":"983126b83f0ae7565666b720c4dde22c9930fd7261ca4bcc8961d600d3ad5a6a","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"be104b66a02409cc4adfb532056ddabf4b40d0214823d8003994a6056e0fd4a4","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"afb4e7bf5663c4264337c9d1e677c0444a5f93e3e544da615e860a878027556d","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","occurred_at":"2026-10-13T15:25:35Z","signature_id":"0af58b2f76d489b90f234310d2a3ac8b142182a0239f813bb77188e32f9dc462","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","signature_id":"c2743ca915206fd985839d671b5618ccb4332bb960f8c549fdcf65d90ddee3d2","pr_number":7491,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"6ab7f7a7d7318820073a94e892ad474901c481d146387431db8c950325a1baa6","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"9578272302c49c5a63e06e79a95c961c925c637a881235d46f047f8f61f08363","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"895d6dc730935dbb7c49cd2e90968b291d7469692edac24ed546ec1e8097382f","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109603048882966528","occurred_at":"2026-10-12T11:12:15Z","signature_id":"bbe6ff38e69c56b0da8c72190329d55c3f58aff3ba835f793e6ee32876e960e6","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109599905499582464","occurred_at":"2026-10-12T10:59:45Z","signature_id":"01ff5d20cf309ae0ced3e300e16a31a96dd5d737827ad14c8718b21686301352","pr_number":7457,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":12,"runs_affected":12,"failed_runs":12,"run_impact_percent":7.2727272727272725,"anchor_occurrences":12,"after_last_push_count":24,"trend_counts":[5,6,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":90,"prior_last_seen_at":"2026-10-14T02:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":12}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T04:04:44Z\n Ended: 2026-10-14T04:10:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7369856-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2teajl722c52pqdr760m7glllbuaiq3i-u9l0s4v1d6h4w4k\", pod=\"cluster-node-tuning-operator-576b745fcc-qpnjx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2teajl722c52pqdr760m7glllbuaiq3i-u9l0s4v1d6h4w4k/cluster-node-tuning-operator-576b745fcc-qpnjx has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T22:19:01Z\n Ended: 2026-10-13T22:28:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2816128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te5m776ut8ij1hl0ecm8bji2rolvh54-y4r5q8u1c0g8x2m\", pod=\"router-5654cb9d4d-gd8l4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te5m776ut8ij1hl0ecm8bji2rolvh54-y4r5q8u1c0g8x2m/router-5654cb9d4d-gd8l4 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T21:38:41Z\n Ended: 2026-10-13T21:44:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3189120-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te4tunmapm3qrg9otufrskbuf4u75vg-h7o1o8e4n0s6g9g\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te4tunmapm3qrg9otufrskbuf4u75vg-h7o1o8e4n0s6g9g/etcd-0 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"17761d523d7af03109fc74ba2a7283ce86a08f29c2f5ee6925a34b270860af93","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"5df3df51c4f1a79ff78be7277fb6033fa17feb7d65772f7a0900bc893660bf39","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","occurred_at":"2026-10-13T20:04:06Z","signature_id":"ab4dcac27ea15b642051449befb4d2d369e9937091c6b496b59f54afa24883cd","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","occurred_at":"2026-10-13T18:01:34Z","signature_id":"920e9198dd57a8245eaf4811aaa20836cb9e3378296216a9590cd685a85e0bbf","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"cd835111d6723302e38161708c21c4a3215a58c651e4b372392909f248560622","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"b0c665dc5593bf939be3f561a3b7d6c563fa192fba4413e100dc7bc2db835e54","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109952299815145472","occurred_at":"2026-10-13T10:20:03Z","signature_id":"153f2db7e7ffaab6160a3f35da43a89c7da37790c513bc78fc32bb4dd3b5f592","pr_number":6944,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"4e6814427594ada40105d0c851f37c68ce6084d56587c92bb86a0a5911644b78","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","signature_id":"36ccc2be84055641dbb6036217141bc4c104ca4b184f2aa2e64ec66e411c6dbe","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"4885c3029699261783a90f8a5a1cf25063711e5d2be18ed2e9aec6344e30e22d","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"eb4f11f09029df76c1cd0c8884283989b59eee33dde5e273ada7dc186fddbab8","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7430/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109519598771507200","occurred_at":"2026-10-12T05:40:39Z","signature_id":"6f8e2d109f354a776a1acb2c7fa60149b6a8d7d0329aa7b37c469a8320c9fe0f","pr_number":7430,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ecdbfeab4fd2cbaa5fe8d221e09bffe86dfcfd6d05ac80416e0ba46ab9b719fc","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"shared-read-only\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":12,"after_last_push_count":0,"trend_counts":[12,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-12T12:06:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should get an existing HCP cluster and report a missing cluster through the RP","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"External auth issuer validation rejects an invalid external auth issuer URL without leaving a child or changing the shared cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Immutable cluster configuration rejects an immutable cluster DNS prefix PATCH without changing the shared cluster","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Immutable node pool configuration rejects an immutable node pool OS disk size PATCH without changing the shared node pool","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Node pool create validation rejects a negative replica count without creating a node pool or changing the shared cluster","occurrences":1}],"full_error_samples":["provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-v4t896dszvwt\": PUT https://rp.j3300864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-v4t896dszvwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-vlx4fgdsmn4s\": PUT https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-vlx4fgdsmn4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------","provision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------\nprovision shared read-only cluster: provision shared node pool: failed starting nodepool creation \"shared-read-only\" for cluster \"shared-read-only\" in resourcegroup=\"shared-ro-lr2d54k7v8w5\": PUT https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shared-ro-lr2d54k7v8w5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shared-read-only/nodePools/shared-read-only\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: MultipleErrorsOccurred\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"shared-read-only\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109616618123300864","occurred_at":"2026-10-12T12:06:11Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7471,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"cc7aa42533f02a48d811cb2a2165e2dbe720082e948a03d214b0799022360b6e","pr_number":7470,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5hsxwfczjwp5\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":6.666666666666667,"anchor_occurrences":11,"after_last_push_count":33,"trend_counts":[3,7,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":117,"prior_last_seen_at":"2026-10-14T00:39:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3899558a41e0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-znvkktb7cpf2\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\",\n \"name\": \"61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T01:44:37.675473518Z\",\n \"endTime\": \"2026-10-14T02:13:43.0652181Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\n correlation ID: 58d6d42f-a8fe-46ff-8c2a-c74cc76252de\n client request ID: ce3ec9eb-4220-47bf-877f-5472abdbec39\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3899558a41e0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-znvkktb7cpf2\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\",\n \"name\": \"61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T01:44:37.675473518Z\",\n \"endTime\": \"2026-10-14T02:13:43.0652181Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4\n correlation ID: 58d6d42f-a8fe-46ff-8c2a-c74cc76252de\n client request ID: ce3ec9eb-4220-47bf-877f-5472abdbec39","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2b2721dce700\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xq96765qgwxz\" to finish creating: GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\",\n \"name\": \"70cccd80-322a-41c6-aef6-dff76071386f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:47:02.133519323Z\",\n \"endTime\": \"2026-10-13T23:16:06.91066671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n correlation ID: 10fb6c26-b492-4ad7-bd1d-956c35d26f43\n client request ID: 62230a28-0708-4526-b5b0-b1e90cbbcfa0\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2b2721dce700\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xq96765qgwxz\" to finish creating: GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\",\n \"name\": \"70cccd80-322a-41c6-aef6-dff76071386f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:47:02.133519323Z\",\n \"endTime\": \"2026-10-13T23:16:06.91066671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n correlation ID: 10fb6c26-b492-4ad7-bd1d-956c35d26f43\n client request ID: 62230a28-0708-4526-b5b0-b1e90cbbcfa0","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1a46c002a160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bvw879vk9kf2\" to finish creating: GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"name\": \"96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:07:49.726995766Z\",\n \"endTime\": \"2026-10-13T22:36:58.709896645Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n correlation ID: 79ec83c3-f7b9-41f5-baf4-9c117904430d\n client request ID: fa7965b1-cdd5-4c6b-bbfc-bfe81d721d6f\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1a46c002a160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bvw879vk9kf2\" to finish creating: GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"name\": \"96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:07:49.726995766Z\",\n \"endTime\": \"2026-10-13T22:36:58.709896645Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n correlation ID: 79ec83c3-f7b9-41f5-baf4-9c117904430d\n client request ID: fa7965b1-cdd5-4c6b-bbfc-bfe81d721d6f"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"ddbe6c18bc9334463938901bbf610b5d5f5e544a53db6228b02093b131657e11","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","occurred_at":"2026-10-13T21:41:52Z","signature_id":"7abba32d1386c404a65777a77a6155bf3cb76d06da7c18bd2cc1819fac47609a","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"3a7b7f1faaa1631909c7c15abf2900e7c8f3a919926de8c9c9b43b605988f0ed","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","occurred_at":"2026-10-13T20:56:53Z","signature_id":"814ddd39348331a533c35437b54ed67f25a7378b56607756b2224dbb1c825092","pr_number":7469,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","occurred_at":"2026-10-13T15:52:02Z","signature_id":"04454a362557e528929a3344b4548e2c84ce0874d9f75e7bb99b09f4d24248b9","pr_number":7313,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","occurred_at":"2026-10-13T15:48:07Z","signature_id":"704940cf6bb6426bf6cc6c262b56027ce5f8ecb93098d3e28c4138854cfe377f","pr_number":7503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110013226245689344","occurred_at":"2026-10-13T14:22:09Z","signature_id":"45e5d5f08ac9de30d727c5c8b1bb049fe1a2cc352bd7a84c7804f7b87b200c41","pr_number":7480,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","occurred_at":"2026-10-13T00:34:34Z","signature_id":"241100486d1e2e5f6bf10dfd23c29984ebe1db934e3f4042fdf918032b8eac38","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109657273612636160","occurred_at":"2026-10-12T14:47:36Z","signature_id":"8a2cb86eeaaac2be5c2b729648eafebf2a78774b306792068dd64d9c83e1f663","pr_number":7311,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","signature_id":"de4fcc56ca6c7e3d2de039fd00e411d285c0c7d0026255cd09fc37bbc4e1d4c7","pr_number":7440,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109474078921330688","occurred_at":"2026-10-12T02:39:47Z","signature_id":"7e05f12cb3af3d2df3800593d5e4d70e1bcc0ddd1495a395607587d40f14bb85","pr_number":7439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":10,"runs_affected":7,"failed_runs":7,"run_impact_percent":4.242424242424242,"anchor_occurrences":10,"after_last_push_count":10,"also_in":["PROD"],"trend_counts":[3,6,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":33,"prior_last_seen_at":"2026-10-14T03:39:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e322e6e67c0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7rswnntg2mwm\" to finish creating: GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\",\n \"name\": \"eeb29541-e4cc-419e-8a9e-17c262940b71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T05:02:26.935820049Z\",\n \"endTime\": \"2026-10-14T05:31:33.678218143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n correlation ID: 91b1a5de-a9bc-4e6e-ad6d-e45a7cb1831a\n client request ID: 4a6ae7e3-e39d-4cda-bb39-3082e7e48f98\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e322e6e67c0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7rswnntg2mwm\" to finish creating: GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\",\n \"name\": \"eeb29541-e4cc-419e-8a9e-17c262940b71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T05:02:26.935820049Z\",\n \"endTime\": \"2026-10-14T05:31:33.678218143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71\n correlation ID: 91b1a5de-a9bc-4e6e-ad6d-e45a7cb1831a\n client request ID: 4a6ae7e3-e39d-4cda-bb39-3082e7e48f98\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fsm9cfscsm2/cluster-candidate-5-0-gcnmsr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x31dfd85b2460\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gcnmsr: failed waiting for cluster=\"cluster-candidate-5-0-gcnmsr\" in resourcegroup=\"rg-candidate-5-0-8fsm9cfscsm2\" to finish creating: GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"name\": \"b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:56:29.235009494Z\",\n \"endTime\": \"2026-10-13T23:25:36.777262748Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n correlation ID: fd6f57aa-c468-479d-a56c-8186a1e8023b\n client request ID: 427c0be1-3302-4f9f-a711-14e8b93546be\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fsm9cfscsm2/cluster-candidate-5-0-gcnmsr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x31dfd85b2460\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gcnmsr: failed waiting for cluster=\"cluster-candidate-5-0-gcnmsr\" in resourcegroup=\"rg-candidate-5-0-8fsm9cfscsm2\" to finish creating: GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"name\": \"b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:56:29.235009494Z\",\n \"endTime\": \"2026-10-13T23:25:36.777262748Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n correlation ID: fd6f57aa-c468-479d-a56c-8186a1e8023b\n client request ID: 427c0be1-3302-4f9f-a711-14e8b93546be\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:125]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x178e97a22840\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-w76dhl6mtjzd\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"name\": \"235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:43.086930646Z\",\n \"endTime\": \"2026-10-13T18:47:49.831018491Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n correlation ID: 077ff34a-ea54-4d78-afd3-f4054328a8bd\n client request ID: 612adb31-4506-4736-9005-28fa95bb228f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:125]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x178e97a22840\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-w76dhl6mtjzd\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"name\": \"235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:43.086930646Z\",\n \"endTime\": \"2026-10-13T18:47:49.831018491Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n correlation ID: 077ff34a-ea54-4d78-afd3-f4054328a8bd\n client request ID: 612adb31-4506-4736-9005-28fa95bb228f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213919732666368","occurred_at":"2026-10-14T03:39:38Z","signature_id":"bccfe50993c9da8f2e5a003a505f4895a9e3eda1fc47df0e84d9a7dfd246b05a","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"3b8fba5a1aa06016e6c6895bc3863f7f6cf359856948127e7c81a165583cfcd6","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"077f299c2a7977d35eb0dc0f0f3806d25b08b958ea3c8d1b9f28717ecf19fa10","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"48f96c794b4248e434313e5383dab85570d91b004b93d8173274ca3b89e3bd5b","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"61bc9fb2baad09f02ca746a494c19b296e155e5562e460538f3b5c55bfa4f3f6","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"afbbb0fd16401b97b4a184026640540584d09079290abbba4ab2babc53541b3a","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"5c9c535151da7212f6c377a91341a3089f208d25d42e32c59743276d5adb4a39","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"ba9594d183dcf679de486f3ef983cb99f7de22b5534227c654d1211082cc3ca0","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"614ca0e1ea00ef1b4414e6ab8b3a11a8b2da5132a3da64d8b9faf88c1203affd","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"026e1543f757852d519080f003f4d7ebec18027795d5835ee8f594400d820c5e","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-5gfztc\" did not complete the SRE-pinned version 4.21.36","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE advances a control plane to an SRE-pinned version","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":4.242424242424242,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[5,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-13T09:11:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE advances a control plane to an SRE-pinned version","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-z4tqzw\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-z4tqzw\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.002s.\ncluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.002s.\ncluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.003s.\ncluster \"sre-version-pin-k9v4gs\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109934998684438528","occurred_at":"2026-10-13T09:11:18Z","signature_id":"e726ff68ce5087e7b46536a414b0c4ce318e72dc425aac96edae158603185df3","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109798012107624448","occurred_at":"2026-10-13T00:06:58Z","signature_id":"17cac32c9f196891eb1cddda2b22294095ac725cb2ad07869f1b1d0d4e898ccd","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","signature_id":"8ec28f71b625b2c11ea506fd0748031d3bf5d074d3279546ef6bfc6cf0d20aaf","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","signature_id":"22c47845096418cb71e4fa35973233fa4b21f339d382954b3aee25b82c29cb84","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","signature_id":"74adfdd35c7ce8eb88d441864d6b1e0ec991a2d255378a9ffce5b6cd5cf8549e","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109629712874606592","occurred_at":"2026-10-12T12:58:13Z","signature_id":"b12b641e87320622b3d715665e73509b0e2928b9b8bc71c938cb2b7200e48d3e","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600098458537984","occurred_at":"2026-10-12T11:00:15Z","signature_id":"f0121469ca4f42ea3b4cc9ff22e2d69e6a7cdd34679fb94a79eb413022a0a715","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":3.6363636363636362,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[4,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":37,"prior_last_seen_at":"2026-10-13T22:12:28Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":6}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T23:20:28Z\n Ended: 2026-10-13T23:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"77588538-1895-449d-bb34-5ee7b9aabbd3\"\n Description: Pod ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-13T23:20:28Z\n Ended: 2026-10-13T23:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"77588538-1895-449d-bb34-5ee7b9aabbd3\"\n Description: Pod ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:51:06Z\n Ended: 2026-10-13T17:10:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0749a532-89d2-4bdf-99d2-f75e8cbffe37\"\n Description: Pod ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-13T16:51:06Z\n Ended: 2026-10-13T17:10:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0749a532-89d2-4bdf-99d2-f75e8cbffe37\"\n Description: Pod ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T21:55:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"cloud-network-config-controller-54f58dd464-7xc6g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d482c703-1325-4911-a179-7fd115deb112\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller-54f58dd464-7xc6g (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T21:56:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"cloud-network-config-controller-54f58dd464-7xc6g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d482c703-1325-4911-a179-7fd115deb112\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/cloud-network-config-controller-54f58dd464-7xc6g (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T21:57:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"csi-snapshot-controller-746f546586-q8jss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8db2a082-8106-487f-b148-74ed6ac93307\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/csi-snapshot-controller-746f546586-q8jss (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-12T21:59:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3618048-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3618048/providers/microsoft.monitor/accounts/hcps-j3618048\", namespace=\"ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o\", pod=\"csi-snapshot-controller-746f546586-q8jss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8db2a082-8106-487f-b148-74ed6ac93307\"\n Description: Pod ocm-arohcpci01-2tdg2k4p3a0tbs6v291jvkejb9gp79a4-s8e1t1g8l8t9e0o/csi-snapshot-controller-746f546586-q8jss (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"ab3005e87ad04ff712d959705038bb2a3a5aadeb7a3d7406d2362bf8f67d8cf4","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"be3f2336b804bb116f4220473f3b514fdb2978302e5dac315a122f4afc722704","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109743222543618048","occurred_at":"2026-10-12T20:29:16Z","signature_id":"d4b2e29eb71d7ac5c954c156d7e1d4a7c4f83854fd89b2f4bf826b82158948d1","pr_number":7457,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"a25524a009142bcd67f70022a288b9e9be2947605657927be7457aeb02bcc1d2","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"ccfda79db149eab1089427f6b12af29fcb8d7b25163196bd92ebcab836f644a6","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"e9c8652d1165ca174ad5201f1063a5d60e6d9e1e776a81a9de4becc5e6a83ae5","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":3.6363636363636362,"anchor_occurrences":6,"after_last_push_count":7,"trend_counts":[3,3,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":40,"prior_last_seen_at":"2026-10-13T22:12:28Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":6}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T23:17:32Z\n Ended: 2026-10-13T23:44:30Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T23:17:32Z\n Ended: 2026-10-13T23:43:31Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:37:29Z\n Ended: 2026-10-13T18:55:26Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:37:29Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T18:38:27Z\n Ended: 2026-10-13T18:55:26Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T18:38:27Z\n Ended: 2026-10-13T18:55:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-13T18:38:27Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-13T18:38:28Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-13T18:38:28Z\n Ended: 2026-10-13T18:55:26Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-13T18:38:28Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:48:12Z\n Ended: 2026-10-13T17:05:12Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-w2fmt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T16:49:11Z\n Ended: 2026-10-13T17:06:14Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-w2fmt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","signature_id":"a9ea08e80c10006881b8dcdd8e6fcea6f11b3f2c9b0ce3779517a03365f66b46","pr_number":7509,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"e9d0a6cb09c8c2cfd53af2a6634e712d86d6ad07e23aa28dffb4ebf4cfd08256","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","signature_id":"812fdff806dd337434b39662e19b80b3cf432a7b1671571c261b606d2d5fc679","pr_number":7485,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"01b14f16705c690af3cc17499286b7047bac4716208b40b6f4d834caa2ce6ebc","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"59230409e21419e206546b3eed7180a2b2d9241870b6db3da89dea2764805770","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"cfbb3abf844cd53f28b64e3663605f410242ad10ab7e9f569183150881111e11","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j0532352 does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":3.6363636363636362,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[1,4,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-14T02:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j0532352 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2816128 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j6775168 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7369856 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7684864 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9626368 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T03:48:00Z\n Ended: 2026-10-14T04:18:05Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T00:46:13Z\n Ended: 2026-10-14T01:18:14Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T22:09:34Z\n Ended: 2026-10-13T22:42:14Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","occurred_at":"2026-10-13T23:39:20Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","occurred_at":"2026-10-13T15:25:35Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[4,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-13T15:32:24Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":3},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert collection is complete","occurrences":3}],"full_error_samples":["alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":3.0303030303030303,"anchor_occurrences":5,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[2,3,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-13T15:48:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d5e48e6be00\u003e: \n failed to create NodePool np-bpvl86: failed waiting for nodepool=\"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\" in resourcegroup=\"rg-hs-pre--94dxdngtsz44\" to finish creating: GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"name\": \"83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:46:39.5702476Z\",\n \"endTime\": \"2026-10-13T17:15:41.984565887Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n correlation ID: 4c25f049-32a9-4598-b0f0-3e14c758ee5b\n client request ID: c16c3d79-0122-4655-b7de-468b0afbb13d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d5e48e6be00\u003e: \n failed to create NodePool np-bpvl86: failed waiting for nodepool=\"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\" in resourcegroup=\"rg-hs-pre--94dxdngtsz44\" to finish creating: GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"name\": \"83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:46:39.5702476Z\",\n \"endTime\": \"2026-10-13T17:15:41.984565887Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n correlation ID: 4c25f049-32a9-4598-b0f0-3e14c758ee5b\n client request ID: c16c3d79-0122-4655-b7de-468b0afbb13d\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28b03e9a6680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-69pgmlxbh7vh\" to finish creating: GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"name\": \"b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T14:11:58.76773642Z\",\n \"endTime\": \"2026-10-13T14:41:02.859999025Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n correlation ID: c7972c9d-9ddb-4652-a9ac-8fbcdb522c95\n client request ID: e9e05827-93d6-4587-bf66-ed9dc69c4222\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28b03e9a6680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-69pgmlxbh7vh\" to finish creating: GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"name\": \"b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T14:11:58.76773642Z\",\n \"endTime\": \"2026-10-13T14:41:02.859999025Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n correlation ID: c7972c9d-9ddb-4652-a9ac-8fbcdb522c95\n client request ID: e9e05827-93d6-4587-bf66-ed9dc69c4222\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x121182295b80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-thbvsvb7b8fq\" to finish creating: GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"name\": \"6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T10:21:53.436571535Z\",\n \"endTime\": \"2026-10-13T10:50:54.288659415Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n correlation ID: a74e8836-2a27-4d03-aee8-222257d39c0c\n client request ID: 47ce1976-0b05-4962-8dcb-6f506b053324\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x121182295b80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-thbvsvb7b8fq\" to finish creating: GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"name\": \"6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T10:21:53.436571535Z\",\n \"endTime\": \"2026-10-13T10:50:54.288659415Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n correlation ID: a74e8836-2a27-4d03-aee8-222257d39c0c\n client request ID: 47ce1976-0b05-4962-8dcb-6f506b053324\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","occurred_at":"2026-10-13T15:48:07Z","signature_id":"0b110e5c27c96f6c961d41c663c021306587e891bbe40929150d9ebda913bc3f","pr_number":7503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","occurred_at":"2026-10-13T13:07:01Z","signature_id":"64c28e7e5c45c72d72e63cae7d8453adeb4e0d6a5ad577f87fac746a2ba5e16b","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109933810454892544","occurred_at":"2026-10-13T09:06:35Z","signature_id":"719b91552139342f4f6f074922a7af2b8c76f3cf3df0a249b92551325953e60c","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109754183480512512","occurred_at":"2026-10-12T21:12:49Z","signature_id":"e7d1ae872e2d1e1f299e0950e3c7121739a7ca3253cb34361fe8fe1de4b27d3a","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109709852522582016","occurred_at":"2026-10-12T18:16:39Z","signature_id":"f07f78c84cfe8e58c2511543b04e0ca7a20c652e502f4d4a311097ea89b1549b","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":3.0303030303030303,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,2,3,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-14T05:04:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x279f2da40c00\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x279f2da40be0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x279f2c9fa820\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x279f2da40bc0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x279f2eb19620\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x279f2da40c00\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x279f2da40be0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x279f2c9fa820\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x279f2da40bc0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x279f2eb19620\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2624930243a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x262493024380\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x2624924923c0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x262493024340\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x26249412ae10\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2624930243a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x262493024380\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x2624924923c0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x262493024340\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x26249412ae10\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f598a00de0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x25f598a00dc0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x25f597dc0730\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x25f598a00da0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x25f59a111950\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f598a00de0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x25f598a00dc0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x25f597dc0730\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x25f598a00da0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x25f59a111950\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"388fa5f884dc9873a3e0ac5e7934c037c1b2aca0708794c9638a9509d5d20e5f","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110213970672488448","occurred_at":"2026-10-14T03:39:51Z","signature_id":"c3f223ea41e95da8d49b0cc04384dc9d6bba824c5592c8a1fc0a741ad927ca6d","pr_number":7504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"d34b77d805ee032e385bd7e9ec93a0f17a9c99b3eb797f5536963db648176fbe","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","signature_id":"694e2d897de6010d0383c8b74dec7d08484cd381d7c5e3824ebcf322dba221fb","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","occurred_at":"2026-10-13T20:56:53Z","signature_id":"efb9c37e965d24b97d7e8ab613bd6ca3432ad4802836e8dcd3a2df165e5003e5","pr_number":7469,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,5,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-14T02:08:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8fmvrw\" to 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x189d56536760\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8fmvrw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n \"name\": \"008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:16:19.986510176Z\",\n \"endTime\": \"2026-10-14T04:00:20.242906036Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21\n client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8fmvrw\" to 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x189d56536760\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8fmvrw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n \"name\": \"008de110-0fc7-4cfb-b1a8-16dd774cc912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:16:19.986510176Z\",\n \"endTime\": \"2026-10-14T04:00:20.242906036Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912\n correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21\n client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v9gdsd\" to 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x89c8a7b4520\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-v9gdsd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lrtt9r887h59\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n \"name\": \"149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:29:27.04612928Z\",\n \"endTime\": \"2026-10-14T04:13:34.922970159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7\n client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v9gdsd\" to 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x89c8a7b4520\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-v9gdsd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lrtt9r887h59\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n \"name\": \"149ad663-a4d9-4ebd-9101-8c188eed65d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:29:27.04612928Z\",\n \"endTime\": \"2026-10-14T04:13:34.922970159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0\n correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7\n client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-trt4r8\" to 4.23\nUnexpected error:\n \u003c*fmt.wrapError | 0xbc2497a0a40\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-trt4r8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nm455m5dz4rg\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\",\n \"name\": \"ab2c099b-e08d-48bf-8615-536073622db3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:17:19.784745543Z\",\n \"endTime\": \"2026-10-14T04:01:20.236478176Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145\n client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-trt4r8\" to 4.23\nUnexpected error:\n \u003c*fmt.wrapError | 0xbc2497a0a40\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-trt4r8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nm455m5dz4rg\" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\",\n \"name\": \"ab2c099b-e08d-48bf-8615-536073622db3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:17:19.784745543Z\",\n \"endTime\": \"2026-10-14T04:01:20.236478176Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3\n correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145\n client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"5d250e50a264778942c74b3ce0da1834bd17192a108f576c77963d2bd5104c98","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"850a794b118cd701611798b45a5bcb03f88a9401ed189a5b8f264b754f82d23a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"8825f796a0bcda6f47b5a66cc92693cc86f0c5ca1b9550e8c9c028e39821e838","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"93eac86ad49dfa02d26b8b40788a25a3f63d761987ff78fe67ff401dea4135e9","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110191032388816896","occurred_at":"2026-10-14T02:08:42Z","signature_id":"e89a0c8726bcc2e849495d81caf51843d12d66f11e5cc3df7b6a10e2304b7d01","pr_number":6968,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j0532352 does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":2.4242424242424243,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,3,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-14T02:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j0532352 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j2816128 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j6775168 does not fire","occurrences":1},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j7369856 does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T03:40:18Z\n Ended: 2026-10-14T03:43:19Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units\nFiring 2:\n State: Resolved\n Started: 2026-10-14T03:52:13Z\n Ended: 2026-10-14T04:06:24Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T00:40:15Z\n Ended: 2026-10-14T01:10:12Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T22:03:21Z\n Ended: 2026-10-13T22:29:18Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110199338117369856","occurred_at":"2026-10-14T02:41:42Z","signature_id":"517b35de6d4d5ec7325035eafd6825972465809bddcc7128c935e22b8d7cb4df","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","occurred_at":"2026-10-13T23:39:20Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","pr_number":7495,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,4,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-13T17:32:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1b7c5f7105e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x1b7c6068e900\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1b7c5f7105e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x1b7c6068e900\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x14a7efc6800\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x14a7fffde80\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x14a7efc6800\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x14a7fffde80\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x194b8fb466e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466c0\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466a0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x194b8fb466e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466c0\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466a0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"5c557e1356eb3b63a8003453927c70609ed1a2e90406457fdd54fc88e1fb53e5","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"6334833f41bf18b21c6f6a2861528f92d6fb485ca2f0d52c7d87545a02b6cbc9","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"c15f77333166c293b45237bf13679bc471c7c7da27a687c44335f6bfc722d028","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"f778b2e59ac991a933120117b78a9c24dc834541831bf4f98f55f52a6dba1078","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[2,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-13T15:52:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d94a08f940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j6jbclq8czt4\" to finish creating: GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"name\": \"f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T17:16:34.496076904Z\",\n \"endTime\": \"2026-10-13T17:45:42.970608528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n correlation ID: fa9a1781-c6d1-4176-bdee-881a0f4c4796\n client request ID: 94e1b016-c599-42c0-a81c-eb85c23ce070\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d94a08f940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j6jbclq8czt4\" to finish creating: GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"name\": \"f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T17:16:34.496076904Z\",\n \"endTime\": \"2026-10-13T17:45:42.970608528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n correlation ID: fa9a1781-c6d1-4176-bdee-881a0f4c4796\n client request ID: 94e1b016-c599-42c0-a81c-eb85c23ce070\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x10bf87371b00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w96jh79xf66q\" to finish creating: GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\",\n \"name\": \"1173880a-8bf8-448d-a470-259a36f4e033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:18:06.878392832Z\",\n \"endTime\": \"2026-10-12T16:47:15.056749067Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n correlation ID: 685034b4-f64b-4ad2-9968-7845cc5d96c5\n client request ID: 20bd3fae-18f5-45e3-8cd8-b2f1ea3f8020\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x10bf87371b00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-w96jh79xf66q\" to finish creating: GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\",\n \"name\": \"1173880a-8bf8-448d-a470-259a36f4e033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:18:06.878392832Z\",\n \"endTime\": \"2026-10-12T16:47:15.056749067Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-6t8p8 to exist\\nMachine v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-w96jh79xf66q--managed/providers/Microsoft.Compute/virtualMachines/v2n6a8e9c9p6q9j-np-1-42p4c-j6mkq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1173880a-8bf8-448d-a470-259a36f4e033\n correlation ID: 685034b4-f64b-4ad2-9968-7845cc5d96c5\n client request ID: 20bd3fae-18f5-45e3-8cd8-b2f1ea3f8020\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x11fa8d633b80\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dmqcn9\" in resourcegroup=\"rg-np-noedge-dmqcn9-8bfmwp2gm92p\" to finish creating: GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"name\": \"5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T03:07:24.415643123Z\",\n \"endTime\": \"2026-10-12T03:36:29.188004519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n correlation ID: 72fd1a70-e0c7-48fa-aa70-d0f6edab139d\n client request ID: 04d73669-7402-48f6-a103-fc2cac633166\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x11fa8d633b80\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dmqcn9\" in resourcegroup=\"rg-np-noedge-dmqcn9-8bfmwp2gm92p\" to finish creating: GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"name\": \"5250eaac-747d-49c2-9ef4-8bb8ba7e096f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T03:07:24.415643123Z\",\n \"endTime\": \"2026-10-12T03:36:29.188004519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-7zfxx to exist\\nMachine m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-dmqcn9-8bfmwp2gm92p-np-ne-dmqcn9--managed/providers/Microsoft.Compute/virtualMachines/m0u0w0c5m8f3v2m-npnoedge-4-20-zjc6m-r229g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7832576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5250eaac-747d-49c2-9ef4-8bb8ba7e096f\n correlation ID: 72fd1a70-e0c7-48fa-aa70-d0f6edab139d\n client request ID: 04d73669-7402-48f6-a103-fc2cac633166\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","occurred_at":"2026-10-13T15:52:02Z","signature_id":"48a02a7f69908511527f6ebb16b479024cc99ba313ab90e8e3951306e211831f","pr_number":7313,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660256270290944","occurred_at":"2026-10-12T14:59:35Z","signature_id":"222aacc360be62a53a3a25918371654d8976e4d02348bfbb0f85f02afd94062c","pr_number":7373,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109460696247832576","occurred_at":"2026-10-12T01:46:36Z","signature_id":"8363386ebe99557c4d0a5f2ab5fc61129ce9cc8c82392c35b5213d30f26dde65","pr_number":7424,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a1a1afad2dc1f0ee985d82cc0c7931dff52aae3d7be3427571c055d66528c857","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[3,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-12T20:55:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f1edda03a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-jdq4zztv647q\" to finish creating: GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"name\": \"97c24699-e347-4966-b180-bf6ae5ff9b03\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:54:28.61846625Z\",\n \"endTime\": \"2026-10-12T21:54:42.231408957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-jdq4zztv647q/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2613760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/97c24699-e347-4966-b180-bf6ae5ff9b03\n correlation ID: 8f0a4fda-224a-436d-84e3-d641535231f1\n client request ID: d44e41d0-633f-4992-a25b-ce6473619f8a\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x10139db40a0\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-v52tck7hh95p\" to finish creating: GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"name\": \"567cdaaa-ef82-48d8-9481-2e386ea53f4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T21:36:04.986032041Z\",\n \"endTime\": \"2026-10-12T21:36:44.722884546Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-v52tck7hh95p/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'ccbf438f-08ec-4e2b-ae4e-219ce7dc3762' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8314496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/567cdaaa-ef82-48d8-9481-2e386ea53f4d\n correlation ID: 7bddac41-f6a3-4695-b95b-13972ea0840e\n client request ID: ab9e631f-f159-4b76-a0df-2b299f43d7d8\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x32753cabb000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lcmmtbwtg46t\" to finish creating: GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"name\": \"b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T09:39:05.150325854Z\",\n \"endTime\": \"2026-10-12T09:39:46.941270418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9143424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b663ffa3-8a99-44d9-a132-5e5b41ecbd2f\n correlation ID: 84029b8a-bf49-4092-9e8c-07ae2562e0b2\n client request ID: 168cc3ec-3b47-4824-95c6-847f6de09edb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109749870842613760","occurred_at":"2026-10-12T20:55:41Z","signature_id":"69699f183ac245894ec15c2257ce86fb84bc1f5bed2cf85b38d4c739dedbd9d1","pr_number":7428,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109742847518314496","occurred_at":"2026-10-12T20:27:46Z","signature_id":"d4b16f343401a26bebbb19567d2aeea68fce56219f43f4bb17023f4763f88712","pr_number":7397,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109563550279143424","occurred_at":"2026-10-12T08:35:17Z","signature_id":"f06a1fb8a7fb87970f3aa02a8622a506c32fd712c86f231325a5e5fce4359407","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[2,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-13T15:32:24Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":3}],"full_error_samples":["alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9114496/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j9114496' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2837760/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j2837760' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","signature_id":"a13b64d94d488082a4a9ffd8fdeb35b5d5d0a71c93381c3fedbe0bba79b488d1","pr_number":7501,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109792838379114496","occurred_at":"2026-10-12T23:46:25Z","signature_id":"7dde37c662e4768f610ecb85052e99f87f88976e1e543acbe408c61888caa61d","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109727639982837760","occurred_at":"2026-10-12T19:27:20Z","signature_id":"3223dd11b35ca881a6905f2213555998ec56974987e667be15a6afbf19cd4f19","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[3,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-12T19:36:16Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T21:30:33Z\n Ended: 2026-10-12T22:07:32Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j6043392-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s\", region=\"westus3\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tdf8lncsu04rjm64fbreumbi4l47bg3-l5g9b5r3v2p1b0s (management cluster ci01-j6043392-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T19:50:18Z\n Ended: 2026-10-12T20:25:22Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2503936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tddoajh9tanmcldflt6gegfe8au9a4r-a8g2t7k0z6k5s9l\", region=\"centralus\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tddoajh9tanmcldflt6gegfe8au9a4r-a8g2t7k0z6k5s9l (management cluster ci01-j2503936-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T19:45:35Z\n Ended: 2026-10-12T20:14:37Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j5649024-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tddn1vgtlc9nve2f6icfn7fqenmjbmm-x2t6l2r1k1q6d2z\", region=\"westus3\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tddn1vgtlc9nve2f6icfn7fqenmjbmm-x2t6l2r1k1q6d2z (management cluster ci01-j5649024-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109729919306043392","occurred_at":"2026-10-12T19:36:16Z","signature_id":"0c18256aec4487d73b73645b8e1ded1390ad9218643d5179523cb223b9edd0b5","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109706138382503936","occurred_at":"2026-10-12T18:01:54Z","signature_id":"1e41fec641cef36701893933a54d28e9a3e41519280905104056febcd0e7dcfb","pr_number":7470,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109705158475649024","occurred_at":"2026-10-12T17:58:00Z","signature_id":"1de0d91a762f7a24003e4b3846ddd93be7eb53f9d5a9baa369ea319d275302ba","pr_number":7471,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.8181818181818181,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[2,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":52,"prior_last_seen_at":"2026-10-13T16:19:32Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":3}],"full_error_samples":["alert fired 17 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-hllvl\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-hllvl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14034537-vmss000000-xc7fq\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14034537-vmss000000-xc7fq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-4k67c\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-4k67c has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-czf22\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-czf22 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14034537-vmss000002-9swqr\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14034537-vmss000002-9swqr has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"hubble-relay-58594b5dd7-nnl6j\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/hubble-relay-58594b5dd7-nnl6j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14034537-vmss000001-8tbkl\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14034537-vmss000001-8tbkl has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-jvbz8\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-jvbz8 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-nn274\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-nn274 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-7rvmk\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-7rvmk has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-c67c4f659-qcdm6\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-c67c4f659-qcdm6 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dbd665f4c-8rntv\", region=\"westus3\", severity=\"warning\"\n Description: Pod maestro/maestro-7dbd665f4c-8rntv has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-c67c4f659-c7v65\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-c67c4f659-c7v65 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dbd665f4c-474rm\", region=\"westus3\", severity=\"warning\"\n Description: Pod maestro/maestro-7dbd665f4c-474rm has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-c67c4f659-4zmsg\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-c67c4f659-4zmsg has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-698d94bbf-pwqx5\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-698d94bbf-pwqx5 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-698d94bbf-7crpb\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-698d94bbf-7crpb has been in a non-ready state for longer than 5 minutes.","alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T10:37:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-vk9xs\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-vk9xs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T10:42:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-64b5859f7f-5glfr\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-64b5859f7f-5glfr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T10:42:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0004864-svc\", component=\"kubernetes-infrastructure\", namespace=\"fleet\", pod=\"fleet-controller-76c4b4c8bc-8gvkr\", region=\"westus3\", severity=\"warning\"\n Description: Pod fleet/fleet-controller-76c4b4c8bc-8gvkr has been in a non-ready state for longer than 5 minutes.","alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14292482-vmss000002-dskt8\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14292482-vmss000002-dskt8 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-12T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14292482-vmss000000-472fv\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14292482-vmss000000-472fv has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-12T10:31:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1804928-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dmqmp\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-dmqmp has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"761776c84216b3b60fba1a6f060188d98d184222bdf626e7b5bac2005be4c5d7","pr_number":7502,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","signature_id":"6e8b49a5caf85ad1a9ceb24783ca3a6237c65bb0981b2afa932bd94b35ce4433","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","signature_id":"b224cee0c66252329215d260009da282cdb5e711becf7fce204c7d1cc07cca6f","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,3,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x223786a70108\u003e: \n resource group \"rg-candidate-5-0-l4hscszf2rxz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-lvqr8d\" in resourcegroup=\"rg-candidate-5-0-l4hscszf2rxz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\",\n \"name\": \"b44ea302-7f97-4969-a60c-eec855852cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:40:42.747363753Z\",\n \"endTime\": \"2026-10-13T17:04:51.623105937Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o054eslakn49l68pthqigeds4ule still exists (deletion dispatched at 2026-10-13T16:40:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n correlation ID: 4bfc0959-e629-40fb-9245-01ec4651bd6b\n client request ID: a781f5bf-303e-4280-ba8a-44cc833b6ce4\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x223786a70108\u003e: \n resource group \"rg-candidate-5-0-l4hscszf2rxz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-lvqr8d\" in resourcegroup=\"rg-candidate-5-0-l4hscszf2rxz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\",\n \"name\": \"b44ea302-7f97-4969-a60c-eec855852cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:40:42.747363753Z\",\n \"endTime\": \"2026-10-13T17:04:51.623105937Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o054eslakn49l68pthqigeds4ule still exists (deletion dispatched at 2026-10-13T16:40:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n correlation ID: 4bfc0959-e629-40fb-9245-01ec4651bd6b\n client request ID: a781f5bf-303e-4280-ba8a-44cc833b6ce4\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x34e38763a138\u003e: \n resource group \"rg-cordon-test-9l4cgfh8mhfl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9l4cgfh8mhfl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"name\": \"84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:39:24.254400328Z\",\n \"endTime\": \"2026-10-13T17:03:31.595626382Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o1agno7465r2vsakhkack392k7ql still exists (deletion dispatched at 2026-10-13T16:39:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n correlation ID: 9b32bec0-6d45-4533-a8ad-d586fd5de17c\n client request ID: 0c2d0a32-aa6c-4c82-a1b0-303746bfb85e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x34e38763a138\u003e: \n resource group \"rg-cordon-test-9l4cgfh8mhfl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9l4cgfh8mhfl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"name\": \"84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:39:24.254400328Z\",\n \"endTime\": \"2026-10-13T17:03:31.595626382Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o1agno7465r2vsakhkack392k7ql still exists (deletion dispatched at 2026-10-13T16:39:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n correlation ID: 9b32bec0-6d45-4533-a8ad-d586fd5de17c\n client request ID: 0c2d0a32-aa6c-4c82-a1b0-303746bfb85e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x39bb325a6558\u003e: \n resource group \"admin-api-serialconsole-dwh828chkx9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-dwh828chkx9c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"name\": \"0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:33:20.010489415Z\",\n \"endTime\": \"2026-10-13T16:57:21.684870881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nldrfocqarmt0r34ht6jph1m9d19 still exists (deletion dispatched at 2026-10-13T16:33:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n correlation ID: d6c08569-5a60-4a0f-bb76-207075ebe9e3\n client request ID: f4713fe8-8c37-423d-a5df-6a24a2af5b2d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x39bb325a6558\u003e: \n resource group \"admin-api-serialconsole-dwh828chkx9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-dwh828chkx9c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"name\": \"0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:33:20.010489415Z\",\n \"endTime\": \"2026-10-13T16:57:21.684870881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nldrfocqarmt0r34ht6jph1m9d19 still exists (deletion dispatched at 2026-10-13T16:33:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n correlation ID: d6c08569-5a60-4a0f-bb76-207075ebe9e3\n client request ID: f4713fe8-8c37-423d-a5df-6a24a2af5b2d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"669ef81600d88868158f50a61a32eb55792068d452440bf21253695791921ce2","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"827a5564727e582e9b07423822965a3f6d9902a3cc857595f1ab3814f76516c1","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"cd7a71f4ca6cd6df4fa5f6e8e1c80b538d136ae3f5c114c7ba728953809926e5","pr_number":7496,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[2,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-12T07:37:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-mpwdzx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x149c10e309a0\u003e: \n DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x149c10e30940\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-74hfg.apps.aro.v8f6d9e5u4q8w4o.16el.j9630464.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-s9x9xg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d3dff18b020\u003e: \n DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d3dff18afe0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fmjrc.apps.aro.q1s9i9u1g0l4i3y.v4x2.j6720256.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109549126109630464","occurred_at":"2026-10-12T07:37:59Z","signature_id":"ebfef50e4f33847feb7608db52e9f990adccd34b42505d1aca31667f15925d12","pr_number":7237,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109538389626720256","occurred_at":"2026-10-12T06:55:20Z","signature_id":"4673534f224bf242b8df8885a07c9c85c02ab3a626129d3db6b153550df27ba6","pr_number":7451,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[1,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x36cea0d80120\u003e: \n resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x36ce9f324380\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-4285twt9lhf2\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x36ce9f324360\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x36cea0a86750\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x36cea0d80120\u003e: \n resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x36ce9f324380\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-4285twt9lhf2\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x36ce9f324360\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x36cea0a86750\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x151218b082b8\u003e: \n resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x151216d07800\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x151218b082b8\u003e: \n resource group \"rg-cluster-back-version-9554sthg8hnz\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9554sthg8hnz\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x151216d07800\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-9554sthg8hnz\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x151216d077a0\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x1512170c3950\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x1512170c3920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-9554sthg8hnz\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x15121775bc50\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-9554sthg8hnz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"ecb5ffa542f87571700262dae2b7c84f4536526d903a70c009dcccac42adf516","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"b46b21369678c0c798aca134326c4166ed0c77d047a753de7c68744c3f55b704","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,1,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-14T00:39:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x18d22f431050\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n correlation ID: d277ac90-384a-44a9-9799-7698621627f9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x18d22f431050\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n correlation ID: d277ac90-384a-44a9-9799-7698621627f9\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x3788bb56a390\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n correlation ID: 135296a0-8b8d-4ae4-9dc1-dcc858885005\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x3788bb56a390\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n correlation ID: 135296a0-8b8d-4ae4-9dc1-dcc858885005\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168597706838016","occurred_at":"2026-10-14T00:39:33Z","signature_id":"e91976aae7db2919970076bbc44f0d95243241475acf02a79c2cb09c59930f22","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","occurred_at":"2026-10-13T20:04:06Z","signature_id":"e835da002c88402841b37e84e6cf6143f8cac4fd26e56433266edb67a38bff39","pr_number":7480,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[1,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-13T10:43:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2219287fa880\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v5gf99nj48r6\" to finish creating: GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\",\n \"name\": \"2199d8bb-869a-41e2-acec-31983e058033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T11:37:06.655702009Z\",\n \"endTime\": \"2026-10-13T12:06:15.607847324Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.18 is Partial (want Completed), started 8m45s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n correlation ID: 436fe8be-4edd-4c43-bff6-c5ba6dee20f5\n client request ID: 88121473-7bcb-4b6c-858a-daddb3fc95db\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2219287fa880\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v5gf99nj48r6\" to finish creating: GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\",\n \"name\": \"2199d8bb-869a-41e2-acec-31983e058033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T11:37:06.655702009Z\",\n \"endTime\": \"2026-10-13T12:06:15.607847324Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.18 is Partial (want Completed), started 8m45s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n correlation ID: 436fe8be-4edd-4c43-bff6-c5ba6dee20f5\n client request ID: 88121473-7bcb-4b6c-858a-daddb3fc95db\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x36d0cb4c81e0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7qkjc46tf97\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"name\": \"d6731312-0cc9-4e1d-a296-bfc0d751168f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:24.479848853Z\",\n \"endTime\": \"2026-10-12T19:14:30.222274838Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 19m29s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d6731312-0cc9-4e1d-a296-bfc0d751168f\n correlation ID: 4572b63a-1060-450d-b5db-9eae5d0fdb78\n client request ID: 5f229b06-fd04-479f-bdad-788fe5458949\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109958187410001920","occurred_at":"2026-10-13T10:43:27Z","signature_id":"7322f74e045c9377f4c356e763fe2af1a5554b1f96d50357c44c7e7728ee62bb","pr_number":7488,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"317dedee167ea472d7b8eb2f6ca44fba55d08af18e00d84e205137d4efe864b6","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a36cdec30df49971b11d0532fa3d90b6d04db004910c6c9ba6b1036a99dab7a3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[2,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T18:38:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x194566f239e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-smsvt85kmkqr\" to finish creating: GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"name\": \"0da32617-05fc-4b99-8fa9-4a08b5bd5119\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:35:51.873467614Z\",\n \"endTime\": \"2026-10-12T20:04:52.500489339Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-4gnlk-rqpw4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-4gnlk-rqpw4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-smsvt85kmkqr--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-4gnlk-rqpw4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2661632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0da32617-05fc-4b99-8fa9-4a08b5bd5119\n correlation ID: 6e6e9b92-3b22-4eef-a78b-96ae21c44dbf\n client request ID: d66a3161-f2e3-4d33-bb73-476a2a9b1435\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bbf606b9160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-frt9kkhsnfm2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"name\": \"75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:36:28.504164951Z\",\n \"endTime\": \"2026-10-12T12:05:35.482369546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qmq6h-h7gmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qmq6h-h7gmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n correlation ID: d89bc198-69ae-4651-acfe-b566b134eb9a\n client request ID: d8b88a68-6fe2-4293-891c-8ad5621c82b5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1bbf606b9160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-frt9kkhsnfm2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"name\": \"75475c40-7fa1-4e8f-bf17-baaab7fc2f88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:36:28.504164951Z\",\n \"endTime\": \"2026-10-12T12:05:35.482369546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qmq6h-h7gmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qmq6h-h7gmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-frt9kkhsnfm2--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qmq6h-h7gmx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/75475c40-7fa1-4e8f-bf17-baaab7fc2f88\n correlation ID: d89bc198-69ae-4651-acfe-b566b134eb9a\n client request ID: d8b88a68-6fe2-4293-891c-8ad5621c82b5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109715328622661632","occurred_at":"2026-10-12T18:38:25Z","signature_id":"0b8a3b5437beafe0105aa62d86153fdc67ea6aaf34efdb9197853845e0812245","pr_number":7479,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"810897085d9afd12e972e9199e65575a09acca1a53e133ff2eb94e81c5444daf","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[1,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-13T00:06:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d502f6120\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-pkbxzb\" in resourcegroup=\"rg-np-version-upgrade-pkbxzb-8r9ncw6xxd9f\" to finish creating: GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"name\": \"d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:17:18.97912251Z\",\n \"endTime\": \"2026-10-13T01:46:20.604608943Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n correlation ID: 4e3d5b66-dbd0-45e8-931a-5c34e6be67a5\n client request ID: 33b8ec2f-ff38-476b-ac0b-f1b2c13ca519\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d502f6120\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-pkbxzb\" in resourcegroup=\"rg-np-version-upgrade-pkbxzb-8r9ncw6xxd9f\" to finish creating: GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"name\": \"d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:17:18.97912251Z\",\n \"endTime\": \"2026-10-13T01:46:20.604608943Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n correlation ID: 4e3d5b66-dbd0-45e8-931a-5c34e6be67a5\n client request ID: 33b8ec2f-ff38-476b-ac0b-f1b2c13ca519\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb6d00db1440\u003e: \n failed to create NodePool np-kxb95g: failed waiting for nodepool=\"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\" in resourcegroup=\"rg-hs-pre--x6fpqw6hqtjb\" to finish creating: GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"name\": \"dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:32:07.510834189Z\",\n \"endTime\": \"2026-10-12T20:01:15.571433072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n correlation ID: bfbdef2c-b02a-48b8-a68a-a2439acfc443\n client request ID: 7e134c20-557c-48a5-9100-cbcf934467d5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xb6d00db1440\u003e: \n failed to create NodePool np-kxb95g: failed waiting for nodepool=\"np-kxb95g\" for cluster \"cluster-hs-pre-kxb95g\" in resourcegroup=\"rg-hs-pre--x6fpqw6hqtjb\" to finish creating: GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"name\": \"dd473c31-c8c5-442b-a6d6-e05341bd54a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T19:32:07.510834189Z\",\n \"endTime\": \"2026-10-12T20:01:15.571433072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2342656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dd473c31-c8c5-442b-a6d6-e05341bd54a5\n correlation ID: bfbdef2c-b02a-48b8-a68a-a2439acfc443\n client request ID: 7e134c20-557c-48a5-9100-cbcf934467d5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","signature_id":"43851cc86df3904c329cff3884669981de4ca71371d688f448f3b74e28255f14","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7373/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109708565542342656","occurred_at":"2026-10-12T18:11:33Z","signature_id":"8d1213d4562f39353e445c7a58b0fd1c57fd1080920c955cf4d994b8f4a3de3c","pr_number":7373,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[2,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-12T11:44:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x2f223f7aa0d8\u003e: \n resource group \"hcp-metrics-lh6cn9z9mpvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-lh6cn9z9mpvb\" to finish deleting: GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"name\": \"666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T13:00:52.055384192Z\",\n \"endTime\": \"2026-10-12T13:24:57.118445449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td8hkede6pik7afuli4rqulo2mdm0ib still exists (deletion dispatched at 2026-10-12T13:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n correlation ID: 32f74568-8662-4b4e-8805-1aefdc453b8a\n client request ID: 470d44cf-4554-4545-aed1-856e45d54cdb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x2f223f7aa0d8\u003e: \n resource group \"hcp-metrics-lh6cn9z9mpvb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-lh6cn9z9mpvb\" to finish deleting: GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"name\": \"666cf1a9-76c6-4e79-8657-ba56dbe93c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T13:00:52.055384192Z\",\n \"endTime\": \"2026-10-12T13:24:57.118445449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td8hkede6pik7afuli4rqulo2mdm0ib still exists (deletion dispatched at 2026-10-12T13:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4879872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/666cf1a9-76c6-4e79-8657-ba56dbe93c28\n correlation ID: 32f74568-8662-4b4e-8805-1aefdc453b8a\n client request ID: 470d44cf-4554-4545-aed1-856e45d54cdb\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0xde424e4090\u003e: \n resource group \"admin-credential-lifecycle-test-7p8rt54bzxxl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-cm6t76\" in resourcegroup=\"admin-credential-lifecycle-test-7p8rt54bzxxl\" to finish deleting: GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\",\n \"name\": \"d217aaa4-6911-41fe-a395-709b350055ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:20:06.890936008Z\",\n \"endTime\": \"2026-10-12T12:44:10.847761888Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td7rmojjop6e5ni0mk3h0r2c95v0aqq still exists (deletion dispatched at 2026-10-12T12:20:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n correlation ID: 5a65ad43-439a-49ac-a5b3-3ea36ef8c6fd\n client request ID: 3b679075-36d2-4af2-ad13-7fdd7af0d4bb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0xde424e4090\u003e: \n resource group \"admin-credential-lifecycle-test-7p8rt54bzxxl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-cm6t76\" in resourcegroup=\"admin-credential-lifecycle-test-7p8rt54bzxxl\" to finish deleting: GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\",\n \"name\": \"d217aaa4-6911-41fe-a395-709b350055ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:20:06.890936008Z\",\n \"endTime\": \"2026-10-12T12:44:10.847761888Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2td7rmojjop6e5ni0mk3h0r2c95v0aqq still exists (deletion dispatched at 2026-10-12T12:20:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5767424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d217aaa4-6911-41fe-a395-709b350055ad\n correlation ID: 5a65ad43-439a-49ac-a5b3-3ea36ef8c6fd\n client request ID: 3b679075-36d2-4af2-ad13-7fdd7af0d4bb\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611255084879872","occurred_at":"2026-10-12T11:44:52Z","signature_id":"f1b1b884321d5b2ac6d5d82d2bb343fddd2a8c66964ab52673d16c25eb8d5697","pr_number":7472,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109600459105767424","occurred_at":"2026-10-12T11:01:58Z","signature_id":"5b41a952bcf6d2d929fc1668bfb207665a25d88b8be5793b80fd4bb0cf28e575","pr_number":7468,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[1,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2c1d48a96060\u003e: \n resource group \"disabled-image-registry-7ddljxcvwjcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7ddljxcvwjcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"name\": \"bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:07.060491044Z\",\n \"endTime\": \"2026-10-13T17:02:11.454465514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n correlation ID: bc63c490-e9f1-483e-a3e0-dbf6c6ddf72f\n client request ID: 93e5d8b4-b42f-4292-ab79-f0ec030e5a84\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2c1d48a96060\u003e: \n resource group \"disabled-image-registry-7ddljxcvwjcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7ddljxcvwjcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"name\": \"bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:07.060491044Z\",\n \"endTime\": \"2026-10-13T17:02:11.454465514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n correlation ID: bc63c490-e9f1-483e-a3e0-dbf6c6ddf72f\n client request ID: 93e5d8b4-b42f-4292-ab79-f0ec030e5a84\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x643438c6138\u003e: \n resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x643438c6138\u003e: \n resource group \"des-encrypt-mfwpss6hqvsd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-mfwpss6hqvsd\" to finish deleting: GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"name\": \"a58b36b6-30fb-4aee-839d-12ae7b447234\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:03:15.661006285Z\",\n \"endTime\": \"2026-10-12T16:27:21.295309747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5076096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a58b36b6-30fb-4aee-839d-12ae7b447234\n correlation ID: 0827ef63-2c1b-4cad-8799-bd4b41067d42\n client request ID: 2c486539-fc98-4112-bf4c-2bdff7d36160\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"a73e237dc986d50343fffd30aca449541788309f42293d44e9183312dbe9d419","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109658742915076096","occurred_at":"2026-10-12T14:53:34Z","signature_id":"4ea81d2e607f663f33592f25d4328ddf786e7ed343f4f03175a81981d5aeb666","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x343c841917c0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ld8wft\" in resourcegroup=\"rg-np-noedge-ld8wft-lrx6cgcqbn7h\" to finish creating: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n correlation ID: ffecaa7d-bd14-4598-8eba-697dbec64dbe\n client request ID: 41765bff-079d-45be-ae54-98a967650838\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x343c841917c0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ld8wft\" in resourcegroup=\"rg-np-noedge-ld8wft-lrx6cgcqbn7h\" to finish creating: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n correlation ID: ffecaa7d-bd14-4598-8eba-697dbec64dbe\n client request ID: 41765bff-079d-45be-ae54-98a967650838\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x23adf9021e40\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-92gd76g6d97g\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n correlation ID: 349f9734-994e-46db-888e-e1caf77e8fb4\n client request ID: 46549edd-5c88-4d3d-b17b-df9c20ad793d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x23adf9021e40\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-92gd76g6d97g\" to finish creating: GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6616960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/32d45c3f-af20-4089-8a66-e8ffd728ab43\n correlation ID: 349f9734-994e-46db-888e-e1caf77e8fb4\n client request ID: 46549edd-5c88-4d3d-b17b-df9c20ad793d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"8d5ca072ced2a86e224e1159c22de688aa029c8ce11fd664e12553f4ed9b8ecd","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109536551896616960","occurred_at":"2026-10-12T06:48:01Z","signature_id":"9b6a19c4bbd8e0fee198397011d61bfa3837fe3c4ea1afea5bc1a3c2f92f5308","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[2,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":20,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:412]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x8ceb7456d30\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23913a80e730\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.007s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23913a80e730\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"f29a7407248c09d9f2d63fc02a535acb501328e35c4bfacc81f04e0e98c0e854","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109660658009116672","occurred_at":"2026-10-12T15:00:29Z","signature_id":"f0b4f980a6b881fe20a87e1354e5792bdbf23c290ec125adfa5d9d993ae815ea","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-13T22:53:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x283328822c40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x283328822c40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:405]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xabc2b9fef0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:405]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xabc2b9fef0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110141907093950464","occurred_at":"2026-10-13T22:53:29Z","signature_id":"b866590f9b46bfea6312f2e0fb04cbc1e34def7eb9a707daec346d608f89ac23","pr_number":7451,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"7728ea8ada183ffb061ff6026d3b0066e425d012b33ecc69f14c383fb512405c","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"96e9adfd4e670ee16cf7899b2fa9e01ef5f2d945f7e2ae1d347a055685469540","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.2121212121212122,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[2,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T09:45:06Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-10-12T10:35:56.708Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:35:57.392Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:57.601Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:35:57.906Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:35:57.907Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:58.041Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:40:59.966Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:40:59.967Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"","time=2026-10-12T10:30:06.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=map[enabled:true]\ntime=2026-10-12T10:30:07.429Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:07.671Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-10-12T10:30:08.147Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:30:08.275Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-10-12T10:35:10.285Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\"\ntime=2026-10-12T10:35:10.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 13/15\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109581128020004864","occurred_at":"2026-10-12T09:45:06Z","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","pr_number":7439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109578665191804928","occurred_at":"2026-10-12T09:35:22Z","signature_id":"3987053007afb2ecf8bd1bd674af68356fc44474cc553867e494bd040f1ce9f4","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d55a541fe54a611e130419c1f169b8c73c34a953ed35defd84c0a24bbe0a88e3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T16:43:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-cjhwhd\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2dcd6debe660\u003e: \n failed to create HCP cluster cluster-zstream-5-0-cjhwhd: failed waiting for cluster=\"cluster-zstream-5-0-cjhwhd\" in resourcegroup=\"rg-zstream-upgrade-5-0-zbcrdfvtst44\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"name\": \"0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:12.815750049Z\",\n \"endTime\": \"2026-10-13T18:47:19.806209332Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n correlation ID: d259ddfa-e906-4fbb-974b-3c977af82a32\n client request ID: be895625-b480-4da8-a968-9e02baae7cef\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-cjhwhd\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2dcd6debe660\u003e: \n failed to create HCP cluster cluster-zstream-5-0-cjhwhd: failed waiting for cluster=\"cluster-zstream-5-0-cjhwhd\" in resourcegroup=\"rg-zstream-upgrade-5-0-zbcrdfvtst44\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"name\": \"0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:12.815750049Z\",\n \"endTime\": \"2026-10-13T18:47:19.806209332Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n correlation ID: d259ddfa-e906-4fbb-974b-3c977af82a32\n client request ID: be895625-b480-4da8-a968-9e02baae7cef\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dllvsm\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x28758e830200\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dllvsm: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dllvsm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kqbmkzfdlvpv\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"name\": \"96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:30.340236179Z\",\n \"endTime\": \"2026-10-13T18:47:39.7699444Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n correlation ID: 8e81f893-e896-41de-bfb5-ecaf7ea09f96\n client request ID: 8d55acf1-8d30-4678-b2f6-7eed037af357\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dllvsm\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x28758e830200\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dllvsm: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dllvsm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kqbmkzfdlvpv\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"name\": \"96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:30.340236179Z\",\n \"endTime\": \"2026-10-13T18:47:39.7699444Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n correlation ID: 8e81f893-e896-41de-bfb5-ecaf7ea09f96\n client request ID: 8d55acf1-8d30-4678-b2f6-7eed037af357\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"be1b7e4b70ae40bb176b235bc31683c57b20ea13e1c6ce838e84dccb47cf9506","pr_number":7248,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"f76417099900daa68e51ebde43fef8462ff51d9cfbe2e05ce62c4518ce93529f","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-x7n4rm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df325fdde20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq\" in resourcegroup=\"rg-candidate-4-23-25wqkbxjvqpm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n correlation ID: 66a77b00-0c74-46a1-bab1-d73fd9784240\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-x7n4rm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df325fdde20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq\" in resourcegroup=\"rg-candidate-4-23-25wqkbxjvqpm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n correlation ID: 66a77b00-0c74-46a1-bab1-d73fd9784240\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-gpscnh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x19351c9c25a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n correlation ID: 73cd0e3a-2347-4fb2-a937-dc27e30c4de2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-gpscnh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x19351c9c25a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n correlation ID: 73cd0e3a-2347-4fb2-a937-dc27e30c4de2\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"0bee45c13efb267807d223dc42fdb6b42774c050c8b11335bf9553e927e7a784","pr_number":7496,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","signature_id":"c9a49014768753cae5a2b15e7de520cc636773dabb5a420f9a78aef34c0146ac","pr_number":7496,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ed30ba2b16f76586a1f98b0129dc60d2a7c651f09c53499a5a21b5f5dff0e754","failure_pattern":"IngressController loadBalancer config is nil","search_query":"IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T22:06:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x284568f46bc0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x284568f46bc0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x374a5e1794b0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x374a5e1794b0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","signature_id":"1dff6fd88e7b67ee0e3d3f634407f5c779eb894a4458935f87ace999c5c319de","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","signature_id":"91cd8349ac9ddd72fa8ca1ae3cea72c2db06bc446d6e997ab59539aff1b80c1e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9310a4a1ba70b92dcd897ea7ccef6ff3d68afd4098649e3461e4758a9ed7df2f","failure_pattern":"VerifyIngressControllerScope(External) timed out after 5m0","search_query":"VerifyIngressControllerScope(External) timed out after 5m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T05:04:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2c6abc8c6320\u003e: \n VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2c6abc8c62a0\u003e{\n msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2c6abc8c6260\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2c6abc8c6320\u003e: \n VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2c6abc8c62a0\u003e{\n msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2c6abc8c6260\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1ba452ab87c0\u003e: \n VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ba452ab87a0\u003e{\n msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ba452ab8760\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1ba452ab87c0\u003e: \n VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ba452ab87a0\u003e{\n msg: \"failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ba452ab8760\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"7d84995ad741c9681c8c3c699502f42032152628033a5acce166137600455589","pr_number":7477,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110235250528358400","occurred_at":"2026-10-14T05:04:24Z","signature_id":"c488b2162ac9532ad63b94efbbfbaac3551ef53bea0886a3d47dcc0d5f92dd8e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-14T00:39:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-m9c8cf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x27a2ab70e4c0\u003e: \n DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x27a2ab70e3e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-m9c8cf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x27a2ab70e4c0\u003e: \n DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x27a2ab70e3e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"ad240022cb81ab8363b3db81cf92564f82d7d9330d100781ceec147c73181eea","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22fbdb35a870cfc5f2c9e8846465e0b7f85e96d6e430537b6f1feed38d45976b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code KeyVaultAccessForbidden; detail message Unable to access key vault resource '\u003curl\u003e' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T11:44:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x14dd1e091060\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-kdz84x79jt45\" to finish creating: GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"name\": \"75f3427d-1a2b-4425-b9b0-7480d4e0b89b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T12:59:06.888844222Z\",\n \"endTime\": \"2026-10-12T13:28:08.456404461Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine des-encrypt-des-np-f8895-79vxf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine des-encrypt-des-np-f8895-gfvrv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine des-encrypt-des-np-f8895-79vxf: Failed: virtualmachine failed to create or update. err: failed to create or update resource des-encrypt-kdz84x79jt45--managed/des-encrypt-des-np-f8895-79vxf (service: virtualmachine): PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-kdz84x79jt45--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-f8895-79vxf\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: KeyVaultAccessForbidden\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"KeyVaultAccessForbidden\\\",\\n \\\"message\\\": \\\"Unable to access key vault resource 'https://cust-kv-knxkedtao2ry2.vault.azure.net/keys/des-encryption-key/3126b361c5834ce5ad1e97723eb63855' to enable encryption at rest. Please grant get, wrap and unwrap key permissions to disk encryption set 'des... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5598976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75f3427d-1a2b-4425-b9b0-7480d4e0b89b\n correlation ID: bd8a92be-4962-42e7-8e34-639e73d20045\n client request ID: 6eda585f-7541-486b-8dad-d5cd134191ff\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109611047785598976","occurred_at":"2026-10-12T11:44:03Z","signature_id":"a902cb53b68f347d3389b746e7a2281dfe680f58c4b7f49f68ca93a281c00759","pr_number":7470,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64c6e11cbb74bffe980698f0ac18939b0cc44bb682a44aa8b5b4f43aee00cd17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T15:06:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mvkgs4\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x2777a0f273c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-mvkgs4: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mvkgs4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-blssk6qzfvws\" to finish creating: GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"name\": \"a49f2790-97bb-4d56-b4a6-e43cb33e230e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:52:53.687204284Z\",\n \"endTime\": \"2026-10-12T16:21:54.576185121Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-policy-controller, kube-controller-manager, openshift-apiserver, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, openshift-controller-manager, konnectivity-agent, olm-operator, packageserver, certified-operators-catalog, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4380032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a49f2790-97bb-4d56-b4a6-e43cb33e230e\n correlation ID: f4045377-6462-4996-b798-a9fd2a096fae\n client request ID: 01fbf192-7565-417e-a2b4-3d30deb4b951\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109662114334380032","occurred_at":"2026-10-12T15:06:56Z","signature_id":"93e4587d665844128088f4ce3a0a7ac0670bc2eccb4898d241b67e1ee37e48d9","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T16:43:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x384da2442800\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65x7ldfvpfrb\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"name\": \"8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:19:32.972571872Z\",\n \"endTime\": \"2026-10-13T18:48:39.815141519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n correlation ID: 7e9c28a1-2d97-4241-837c-65dbd7be5ebd\n client request ID: 76ea1b85-c87c-48c0-b8b1-76222a78e914\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x384da2442800\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65x7ldfvpfrb\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"name\": \"8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:19:32.972571872Z\",\n \"endTime\": \"2026-10-13T18:48:39.815141519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n correlation ID: 7e9c28a1-2d97-4241-837c-65dbd7be5ebd\n client request ID: 76ea1b85-c87c-48c0-b8b1-76222a78e914\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"b136a9b186b73309fc330d90067542e4a0acbb1b86014bb556a7bc37a5f5c48c","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de8a7215be9cbe7df7ff3c38f8eba50243fbc0204a90552d4bf737d8d09cb701","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T11:41:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-w5zxw6n76pst/cluster-candidate-4-20-dnk6t4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x4d01c384420\u003e: \n failed to create HCP cluster cluster-candidate-4-20-dnk6t4: failed waiting for cluster=\"cluster-candidate-4-20-dnk6t4\" in resourcegroup=\"rg-candidate-4-20-w5zxw6n76pst\" to finish creating: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"name\": \"d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:39:39.471658263Z\",\n \"endTime\": \"2026-10-13T13:08:48.195755915Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n correlation ID: 6fd24ae1-3485-4cf6-8f00-56a00457a1d3\n client request ID: 52d0a7ef-5194-4884-9aba-b6475a9fa220\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-w5zxw6n76pst/cluster-candidate-4-20-dnk6t4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x4d01c384420\u003e: \n failed to create HCP cluster cluster-candidate-4-20-dnk6t4: failed waiting for cluster=\"cluster-candidate-4-20-dnk6t4\" in resourcegroup=\"rg-candidate-4-20-w5zxw6n76pst\" to finish creating: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"name\": \"d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:39:39.471658263Z\",\n \"endTime\": \"2026-10-13T13:08:48.195755915Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n correlation ID: 6fd24ae1-3485-4cf6-8f00-56a00457a1d3\n client request ID: 52d0a7ef-5194-4884-9aba-b6475a9fa220\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","occurred_at":"2026-10-13T11:41:15Z","signature_id":"c12ed8153f187823b5da8c9f2feb1648642f6798384985cc1969cf91f57dcead","pr_number":7485,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27aef5ac99f567baefaad85029e0d3da9b4dbe40c6bbe0be985920a793730d4f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x495f5a50100\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vtqpftbtl8pd\" to finish creating: GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"name\": \"b568c3f0-0cc4-47ab-8959-64f2a472fc67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T16:05:12.565546726Z\",\n \"endTime\": \"2026-10-12T16:34:19.299056423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterValidation] DataPlaneIdentitiesPermissionsValidation: InternalError: Unable to verify data plane identities permissions.; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9626368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b568c3f0-0cc4-47ab-8959-64f2a472fc67\n correlation ID: 2f0e8af3-4477-4dd1-b4a2-0bb21a3ecc71\n client request ID: af43345c-178c-4f70-985a-42472759437b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"59da2ca63ee261235a31615de260be397d214b0f4f694b160b2ac972a17ed31e","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b89623a31d2776290135b67a1c3e8137de86d30ee19f034e5531fb48ad00d00f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T17:55:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0xe13d9ba2880\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-9czhr6hdgkds\" to finish creating: GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"name\": \"abdd1b6c-8284-4503-9367-9545ab4b7e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T18:45:22.752877991Z\",\n \"endTime\": \"2026-10-12T19:14:30.175613398Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6169984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/abdd1b6c-8284-4503-9367-9545ab4b7e3c\n correlation ID: e99f3924-5824-4d14-bfb3-a5c2c635c6fb\n client request ID: 3e3fb9a9-7a4c-4e3b-9ae6-64fae2a5931a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"01fffc7b6758239c11fdf9bc2bb6322f5a22ddef4368d43104f752d3e34ce9e2","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2a17ea6624b520ae513ca4725852ca83305c70a471a12e833ed2d7d18cfe8aaa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T00:39:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x147375bd60d8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l8lntszfbqhk\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\",\n \"name\": \"1bccabc3-2126-4728-9d49-4eea01285f0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T01:45:21.225915122Z\",\n \"endTime\": \"2026-10-14T02:14:22.961369975Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e\n client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x147375bd60d8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l8lntszfbqhk\" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\",\n \"name\": \"1bccabc3-2126-4728-9d49-4eea01285f0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T01:45:21.225915122Z\",\n \"endTime\": \"2026-10-14T02:14:22.961369975Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a\n correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e\n client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168566228586496","occurred_at":"2026-10-14T00:39:25Z","signature_id":"d1401fd1962c2b0a341d2d8996ab42716af9f4edd450ddb234863dd75ed0017e","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9da4ad2c21fc0e265207db4c5abcfdd0631935b407c3edcd41248ee669719f25","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T10:14:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x361e46553da0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dx8x9nb92ts2\" to finish creating: GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"name\": \"09be8631-a40d-4148-b3b2-3144216d4fe7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T11:44:57.797440876Z\",\n \"endTime\": \"2026-10-12T12:14:05.478563828Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-48lvb-qrq6s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-48lvb-qrq6s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dx8x9nb92ts2--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-48lvb-qrq6s to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4754816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/09be8631-a40d-4148-b3b2-3144216d4fe7\n correlation ID: 06eef5ee-18e5-459b-b472-73166e064f53\n client request ID: c3ba459a-88fd-41d3-aa0f-d4135fceec1f\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"6ca522238814ece9529f702529a6bcb452636d575084a8828e41559f858abbbd","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"531a2084e8686f9a42755ac056f2dd31faa67fc9e5c5979d05857ee0dfb1b975","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T17:55:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd4464611460\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xd4464611460\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-b4xvfj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2hklr4tmfmv6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/12/2026 6:44:17 PM' with correlationId '483786d1-9a79-473c-9660-1216b08e7f90', and will expire at '10/19/2026 6:44:17 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-b4xvfj/operationStatuses/08584097762288469100\n correlation ID: d9670e5b-9049-4df2-b486-a7bed21f23b8\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109704412896169984","occurred_at":"2026-10-12T17:55:03Z","signature_id":"01033c5c618998eef4c756c29b116f1c2e1c20ad91b1f1d16200cebb174aafe7","pr_number":7467,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T12:05:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3398cf888a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-qzrlv9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Authorization/roleAssignments/adc91f81-bb61-5a2b-b4cc-0e8c1c460b92' at line '1' and column '27135'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 27135,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n correlation ID: 9860797c-226a-442a-b3bd-c7b7c62b766b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3398cf888a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-qzrlv9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Authorization/roleAssignments/adc91f81-bb61-5a2b-b4cc-0e8c1c460b92' at line '1' and column '27135'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 27135,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n correlation ID: 9860797c-226a-442a-b3bd-c7b7c62b766b\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109978799176159232","occurred_at":"2026-10-13T12:05:21Z","signature_id":"0f4249b820a1949de5b8be0a7d0f2992e157ed0020fa38fe47a583da35703e09","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RequestTimeout; detail message Database account creation failed. Operation Id: \u003cuuid\u003e, Error : Message: {\"Errors\":[\"The requested operation exceeded maximum alloted time. Learn more: \u003curl\u003e; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T23:35:23Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-10-12T23:56:30.391Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-12T23:56:32.064Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-12T23:56:32.955Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3444352%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3\ntime=2026-10-13T00:03:04.471Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3444352/providers/Microsoft.Resources/deployments/2f9fa65a0ba704015b11015a72cc4e4b4675fce22bd699e4c355e7650345c6e3/operationStatuses/08584097574930206657\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3444352/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3444352/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j3444352\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 42664daf-4e8e-4916-9d75-23b7169468da, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 26eaccdb-de17-4cbf-afc1-0c865cfbb1a2, Request URI: /apps/de70b74f-b005-4de6-b2c3-b39046bf9084/services/74353989-92b9-4a77-a49a-b47600be8b14/partitions/a40f5539-0cc6-43c1-b5bc-fdb8dcb67485/replicas/134362021132042649s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-13T00:02:20.9415661Z, RequestEndTime: 2026-10-13T00:02:26.9109610Z, Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:28.0336227Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.273,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21877192.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1501,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:38.0493247Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.521,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21840180.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0747,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:48.0505987Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21844420.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1332,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:01:58.0664654Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.975,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21830384.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1746,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:02:08.0821422Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21833552.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0846,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":48},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:02:18.0977872Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":2.281,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21805124.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.1832,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":51}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-13T00:02:20.9415661Z; ResponseTime: 2026-10-13T00:02:26.9109610Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be424.documents.azure.com:11306/apps/de70b74f-b005-4de6-b2c3-b39046bf9084/services/74353989-92b9-4a77-a49a-b47600be8b14/partitions/a40f5539-0cc6-43c1-b5bc-fdb8dcb67485/replicas/134362021132042649s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 5975.386, ActivityId: 26eaccdb-de17-4cbf-afc1-0c865cfbb1a2, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11306 | status: Connected | lkt: 10/13/2026 12:02:26 AM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415661Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0013},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415674Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.006},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9415734Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0628},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:20.9416362Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 5976.0734},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:26.9177096Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.1891},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-13T00:02:26.9178987Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:07.6413599Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:07.6413599Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-13T00:00:12.9085149Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109790061913444352","occurred_at":"2026-10-12T23:35:23Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7483,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-13T19:57:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-5wrsns\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x9f731ad0ea0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-mve2kajh3vwg6' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n correlation ID: 547339b0-2310-46d9-9b61-b7c398526e80\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-5wrsns\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x9f731ad0ea0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-mve2kajh3vwg6' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n correlation ID: 547339b0-2310-46d9-9b61-b7c398526e80\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","signature_id":"c26a9d223de96e2bc27095ea2774785d4152120405397e9ea2ecd9e4d14ed85b","pr_number":7503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-13T19:57:04Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-13T10:18:54.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n Kind: ARM\\n Template: templates/mgmt-infra.bicep\\n Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-10-13T10:18:55.539Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-10-13T10:18:56.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=fa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0355584-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1\ntime=2026-10-13T10:20:27.529Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0355584-mgmt-2/providers/Microsoft.Resources/deployments/fa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1/operationStatuses/08584097201492563595\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0355584-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0355584-2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j0355584-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109943465260355584","occurred_at":"2026-10-13T09:44:57Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-12T14:42:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ec259084840\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-tv5qjjvw6mt5\" to finish creating: GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"name\": \"639bbef1-ebbd-4367-896d-cec3c0263d16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T15:42:56.655438798Z\",\n \"endTime\": \"2026-10-12T15:43:02.11596193Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/639bbef1-ebbd-4367-896d-cec3c0263d16\n correlation ID: 1ceff82d-31f8-4b03-ac69-9b0cd1c3b903\n client request ID: 24f2f909-e5ab-41c2-8601-8e5d6aea83f8\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109655959428141056","occurred_at":"2026-10-12T14:42:30Z","signature_id":"0b00e0911be997c70989e7dd3678d4edc17e5aca2336e84d6fe6cf814fadeb28","pr_number":7440,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7b00d884b9121fd268447d271afcfe83e409041219994587880e368f74202f32","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T11:41:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x12402d3b2b10\u003e: \n resource group \"rg-candidate-4-23-qt7g4z697d5b\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-ch4n9f\" in resourcegroup=\"rg-candidate-4-23-qt7g4z697d5b\" to finish deleting: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"name\": \"27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:59:59.269440518Z\",\n \"endTime\": \"2026-10-13T13:24:08.339110012Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdtgm7u9ipde1epc2rt5pe43isp14m7 still exists (deletion dispatched at 2026-10-13T12:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n correlation ID: 365de814-fd9f-4b17-8525-a2096af1cada\n client request ID: 166f0483-5054-4255-b484-02e488295af5\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x12402d3b2b10\u003e: \n resource group \"rg-candidate-4-23-qt7g4z697d5b\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-ch4n9f\" in resourcegroup=\"rg-candidate-4-23-qt7g4z697d5b\" to finish deleting: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"name\": \"27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:59:59.269440518Z\",\n \"endTime\": \"2026-10-13T13:24:08.339110012Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdtgm7u9ipde1epc2rt5pe43isp14m7 still exists (deletion dispatched at 2026-10-13T12:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n correlation ID: 365de814-fd9f-4b17-8525-a2096af1cada\n client request ID: 166f0483-5054-4255-b484-02e488295af5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","occurred_at":"2026-10-13T11:41:15Z","signature_id":"ed359cc4388f74eb3c414e5bd2ce69baff8941bc6e08595174f886722dcbacee","pr_number":7485,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T23:59:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x1a1cb20b4078\u003e: \n resource group \"feature-aggregation-vthdss5fr9gf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vthdss5fr9gf\" to finish deleting: GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n \"name\": \"a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:31:01.090497823Z\",\n \"endTime\": \"2026-10-13T01:55:09.444868511Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n correlation ID: 77f162b5-13b3-4f68-9936-971476bd0866\n client request ID: 9306570d-d008-4fe7-9f11-7b82a031625e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x1a1cb20b4078\u003e: \n resource group \"feature-aggregation-vthdss5fr9gf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vthdss5fr9gf\" to finish deleting: GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n \"name\": \"a499b952-55a7-42b0-9b5b-9062dcf2e20d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:31:01.090497823Z\",\n \"endTime\": \"2026-10-13T01:55:09.444868511Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7859840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a499b952-55a7-42b0-9b5b-9062dcf2e20d\n correlation ID: 77f162b5-13b3-4f68-9936-971476bd0866\n client request ID: 9306570d-d008-4fe7-9f11-7b82a031625e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109796058077859840","occurred_at":"2026-10-12T23:59:12Z","signature_id":"640722fdcf6ffb0b7a84d5246e5db08c2b424ef7488f0c86ca3478e9b885ef75","pr_number":7484,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-13T19:57:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x332d8f70c120\u003e: \n resource group \"rg-hs-pre--pvfm6ns8qtjf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6j9rv6\" in resourcegroup=\"rg-hs-pre--pvfm6ns8qtjf\" to finish deleting: GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"name\": \"8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T21:38:32.56326477Z\",\n \"endTime\": \"2026-10-13T22:02:34.567780855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n correlation ID: c238bf11-ea0d-4948-b2da-1eaa7be46473\n client request ID: bbe62e16-afcb-49d8-bc81-105532d0b70b\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x332d8f70c120\u003e: \n resource group \"rg-hs-pre--pvfm6ns8qtjf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6j9rv6\" in resourcegroup=\"rg-hs-pre--pvfm6ns8qtjf\" to finish deleting: GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"name\": \"8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T21:38:32.56326477Z\",\n \"endTime\": \"2026-10-13T22:02:34.567780855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n correlation ID: c238bf11-ea0d-4948-b2da-1eaa7be46473\n client request ID: bbe62e16-afcb-49d8-bc81-105532d0b70b\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","signature_id":"181ae70b8645dc86d286ecef9ed77dae1b64d0e1fdc182c681d11979a0362e0d","pr_number":7503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-12T13:54:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x72d88398300\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x72d88398300\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2m7pgwml79qx\": PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://rp.j4532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2m7pgwml79qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n correlation ID: 0548989b-5739-4134-ae92-44e0b94372c4\n client request ID: eff3cdde-c6fa-4be6-bf0f-91fa006cb430\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109643872534532096","occurred_at":"2026-10-12T13:54:09Z","signature_id":"1f4d7a78c6e6f2b64bfcac7b4fc7dadaf9ee3793c7757ef0699936b0dd1911ad","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T19:26:38Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-13T20:10:59.938Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-13T20:11:00.569Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-533168303/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091971564408832","occurred_at":"2026-10-13T19:26:38Z","signature_id":"9ca8c9ef2c96a2d57799808a4c23411872eadb74fb13e040cc050d45a0ce4d60","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35f4b32d0c9a217f6e6dcfd9cb816a96a3bf1d26a1497dfd78708f5764de5a24","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Verify cluster operators should provision an HCP cluster and verify the cluster operators","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T11:58:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Verify cluster operators should provision an HCP cluster and verify the cluster operators","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_operators.go:163]: failed to verify cluster operators on cluster \"cluster-operators-qwqhtt\"\nUnexpected error:\n \u003c*errors.joinError | 0x275bb568108\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node\n {\n errs: [\n \u003c*fmt.wrapError | 0x275bb4fe480\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78f20\u003e{\n msg: \"EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78d80\u003e{\n msg: \"timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*errors.errorString | 0x275bb7584c0\u003e{\n s: \"EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_operators.go:163]: failed to verify cluster operators on cluster \"cluster-operators-qwqhtt\"\nUnexpected error:\n \u003c*errors.joinError | 0x275bb568108\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node\n {\n errs: [\n \u003c*fmt.wrapError | 0x275bb4fe480\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78f20\u003e{\n msg: \"EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78d80\u003e{\n msg: \"timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*errors.errorString | 0x275bb7584c0\u003e{\n s: \"EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","signature_id":"bd20bcea67709cdf0163960d867a7fb6af562bc860fdce2d4fddc185e5ee2f10","pr_number":7463,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0a9bcf49d179171c63dccafd8dbb4e653af27553fa78df236def98914f6ada90","failure_pattern":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T16:01:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df811d0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df140c0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019e2caff0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xb019e0de2e8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xb019c86fe00\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c86fda0\u003e{\n msg: \"failed to list services: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df811d0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xb019c4d6ff0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019bfab1e0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019bfab0e0\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019df140c0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xb019c5e2a50\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xb019c4e0980\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xb019c4e0960\u003e{\n msg: \"failed to get default IngressController: Get \\\"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\\\": dial tcp: lookup api.private-ingress.9krg.j0754432.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xb019e2caff0\u003e{\n Op: \"Get\",\n URL: \"https://api.private-ingress.9krg.j0754432.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\",\n Err: \u003c*net.OpError | 0xb019bfc47d0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xb019c4d6eb0\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.18.106:35378-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.private-ingress.9krg.j0754432.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109675831700754432","occurred_at":"2026-10-12T16:01:28Z","signature_id":"a87f271596a6f4e182a266365db74092bd67d3130b10c9e295b2604bd480ee54","pr_number":7162,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":15,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":46,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n {\n errs: [\n \u003c*fmt.wrapError | 0x11d596138740\u003e{\n msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n },\n },\n \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0x11d59617f5d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n {\n errs: [\n \u003c*fmt.wrapError | 0x11d596138740\u003e{\n msg: \"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n err: \u003c*errors.errorString | 0x11d59546a3c0\u003e{\n s: \"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\",\n },\n },\n \u003c*fmt.wrapError | 0x11d5961388a0\u003e{\n msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n err: \u003c*errors.errorString | 0x11d59546a6c0\u003e{\n s: \"clusterversion status.history has no version in target minor \\\"4.21\\\"\",\n },\n },\n ],\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"e72a8b419adb6b53b3d96517db9d967450746716b4acc806c6d35e614fc4cc81","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-13T00:06:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x18ba92cf6f8\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x18ba9177500\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba9176100\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba91760e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x18ba92cf6f8\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x18ba9177500\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba9176100\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba91760e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","signature_id":"ea8a7a2dfe743c69bf60d5e3342f070fda004e14eff8a761750c50c706798fa0","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5207ee83cecbb4a62f10c9755387275f2a4075a267928e8754031a9cdbdeb408","failure_pattern":"after 4 attempts: 1 releases found in \u003curl\u003e which is not enough for the requested 1 offset","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T17:32:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:134]: failed to resolve configured channel candidate-5.1\nUnexpected error:\n \u003c*fmt.wrapError | 0xda3675c58e0\u003e: \n after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\n {\n msg: \"after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n err: \u003c*errors.errorString | 0xda366e3dab0\u003e{\n s: \"1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:134]: failed to resolve configured channel candidate-5.1\nUnexpected error:\n \u003c*fmt.wrapError | 0xda3675c58e0\u003e: \n after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\n {\n msg: \"after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n err: \u003c*errors.errorString | 0xda366e3dab0\u003e{\n s: \"1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","signature_id":"2ff884e11f43410ad920aa9b4baf8cb219805fce930bc1a46992b3fb5753fbd3","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":30,"prior_last_seen_at":"2026-10-13T19:02:36Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T20:14:21Z\n Ended: 2026-10-13T20:56:21Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6273152-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j6273152-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","occurred_at":"2026-10-13T19:02:36Z","signature_id":"6f5244fc0fea589d70c2fa4e58a09e522022dcd0de6fff81e3ffbe8c544084ae","pr_number":7505,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-13T16:43:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 12 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:51:01Z\n Ended: 2026-10-13T18:58:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:51:02Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T18:51:02Z\n Ended: 2026-10-13T18:59:03Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fpcp41do03q2ch5ajhtin6o2onk6\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2fpcp41do03q2ch5ajhtin6o2onk6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T18:51:02Z\n Ended: 2026-10-13T18:58:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:58:01Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:58:01Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-13T18:52:58Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-13T18:52:58Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-13T18:54:01Z\n Ended: 2026-10-13T19:01:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2gdnt0p29rhn6u0lod1kmga0s4ejn\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2gdnt0p29rhn6u0lod1kmga0s4ejn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","signature_id":"ff049ca30612ae3c5d981f61312a933989a2df4db1c20adc6d1b9ff278e63fd3","pr_number":7248,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-10-05","2026-10-12"],"prior_runs_affected":13,"prior_last_seen_at":"2026-10-13T16:19:32Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:24:20Z\n Ended: 2026-10-13T18:31:19Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4029440-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:24:20Z\n Ended: 2026-10-13T18:31:19Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4029440-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"26ad01559a977d2ccbf9de49f0e0a12ad85e844d43c5c076830f46ba6268215f","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","failure_pattern":"alert [svc] MaestroServerNoReadyReplicas fired","search_query":"alert [svc] MaestroServerNoReadyReplicas fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T16:19:32Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T18:40:06Z\n Severity: Sev2\n Labels: alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j4029440-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4029440/providers/microsoft.monitor/accounts/services-j4029440\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5lj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T18:40:06Z\n Severity: Sev2\n Labels: alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j4029440-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4029440/providers/microsoft.monitor/accounts/services-j4029440\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5lj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"61b46f21ba4b01a9a3f1d792838519213b86b988924028497bb2806fe1283487","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":22,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-12T16:40:12Z\n Ended: 2026-10-12T16:50:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-56bb7dd747-h9s6h\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-56bb7dd747-h9s6h (namespace aro-hcp) on cluster ci01-j9626368-svc has averaged 134.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-10-12T16:40:12Z\n Ended: 2026-10-12T16:54:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-c6ff9b6f4-b8fh5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-c6ff9b6f4-b8fh5 (namespace kube-applier) on cluster ci01-j9626368-mgmt-2 has averaged 150.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-10-12T16:40:12Z\n Ended: 2026-10-12T16:56:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9626368-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-65fd757c56-d7zhs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-65fd757c56-d7zhs (namespace kube-applier) on cluster ci01-j9626368-mgmt-1 has averaged 164.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"2ac18c0f28c3db988430e8cd508afbb6dc2ac259140c6a1cba13d912134a5113","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T17:20:12Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"d53a2236830f49d3e673f190f6889ac45bcba408a0f6c004202adfc7dcecbe92","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-12T17:06:14Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j9626368-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9626368/providers/microsoft.monitor/accounts/services-j9626368\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j9626368-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"f0a5ca335c5452170de0c7d7ae55e4a8594a4284ebc53b77ca227c096a57881b","pr_number":7477,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-13T19:02:36Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T20:38:20Z\n Ended: 2026-10-13T21:03:20Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j6273152-svc\", component=\"slo\", long_window=\"6h\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j6273152-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","occurred_at":"2026-10-13T19:02:36Z","signature_id":"751897ff3429e507ba41ef73f8187f9a0987de78f57b98795b44735681e9a017","pr_number":7505,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1b9753d6df9613da46b4b9cfaf32730b876169734288bd96a4ec92c7f1a772fa","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T10:14:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-82f6cfwmh699'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x258297402990\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2582980c0060\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\n let _endTime = datetime(2026-10-12T11:53:04.2658821Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-82f6cfwmh699'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-12T11:53:10.3021968Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\", ActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", SubActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"583de96c-d634-44b3-9661-ce011792b0cc\", ActivityStack:\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x258297402990\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x2582980c0040\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582980c0020\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x2582961135c0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x258296113580\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-11T11:53:04.2658819Z);\\nlet _endTime = datetime(2026-10-12T11:53:04.2658821Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-82f6cfwmh699'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-12T11:53:10.3021968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;48584352-556a-4a9a-886f-653d24baecf1\\\", ActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", SubActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"583de96c-d634-44b3-9661-ce011792b0cc\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;48584352-556a-4a9a-886f-653d24baecf1 ARID=583de96c-d634-44b3-9661-ce011792b0cc \u003e GW.Http.CallContext/583de96c-d634-44b3-9661-ce011792b0cc)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x258296cdc200\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109588388414754816","occurred_at":"2026-10-12T10:14:00Z","signature_id":"670cbec864bb9c486cb9441591ed8f63fe3036fd2cb244750889987cfff959c9","pr_number":7420,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"deea4d7ee03e8432bcb988590b2236f74b08241de4cdde1789d1247ae2f6a03f","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T19:28:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39ffd3acaac0\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\n let _endTime = datetime(2026-10-13T21:06:23.2601111Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T21:06:33.6034532Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\", ActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", SubActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityStack:\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39ffd2546540\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39ffd3acaaa0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd3acaa80\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047e00\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047dc0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39ffd0960100\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39ffd3acaac0\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\n let _endTime = datetime(2026-10-13T21:06:23.2601111Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T21:06:33.6034532Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\", ActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", SubActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityStack:\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39ffd2546540\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39ffd3acaaa0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd3acaa80\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047e00\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047dc0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39ffd0960100\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091526691360768","occurred_at":"2026-10-13T19:28:19Z","signature_id":"f25671b9f4bd359b018cb9e9ac954a66d4ed6290a26895e1fd925ac9f22976fe","pr_number":7492,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8baf31258527a9c37481b02b6c4e240b9ba893342f978c453787fc7905c9d95b","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T21:41:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 604.992s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39f5f91e0c80\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\n let _endTime = datetime(2026-10-13T23:09:06.1094687Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T23:09:29.6842855Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\", ActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", SubActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39f5f8768288\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39f5f91e0c60\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0c40\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0ba0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0b60\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39f5f6b7ee00\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 604.992s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39f5f91e0c80\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\n let _endTime = datetime(2026-10-13T23:09:06.1094687Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T23:09:29.6842855Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\", ActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", SubActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39f5f8768288\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39f5f91e0c60\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0c40\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0ba0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0b60\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39f5f6b7ee00\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123903912251392","occurred_at":"2026-10-13T21:41:57Z","signature_id":"8b0aac51428f6336203e5c99b582b7417a3d1fd3f1323459c75ebe8411a4f273","pr_number":7488,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"38232e514c86bfe6cab965d283fee05c3ce21024c2a543efb679b24a42cf1009","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T00:39:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1e936bab0320\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\n let _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x1e936b180db0\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x1e936bab0300\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1e936bab02e0\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1e93699ddd20\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1e93699ddc60\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x1e936a2c1400\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1e936bab0320\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\n let _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x1e936b180db0\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x1e936bab0300\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1e936bab02e0\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1e93699ddd20\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1e93699ddc60\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-zxm6njn97hd4'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T02:13:17.5036831Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\\\", ActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", SubActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"15fc144c-f310-411c-9ec4-c55a211606b5\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 \u003e GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x1e936a2c1400\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110168614748295168","occurred_at":"2026-10-14T00:39:37Z","signature_id":"83eb0b332787f68fc85dd200d810732ea20732bc14eafa3beefcaaf8deb483c0","pr_number":7509,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b6abc857fe5890451778a1e8bd584c9cdff0daecc370ddea658871d7ad261b7b","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T02:04:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1fce6bc96f40\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\n let _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x1fce6bb2e558\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x1fce6bc96f20\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1fce6bc96f00\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1fce6b3e4d40\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1fce6b3e4d00\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x1fce6d014600\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1fce6bc96f40\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\n let _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x1fce6bb2e558\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x1fce6bc96f20\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1fce6bc96f00\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1fce6b3e4d40\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x1fce6b3e4d00\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sbj59nsxlhq5'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-14T03:48:19.7777147Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\\\", ActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", SubActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"b81cad1f-eefa-4a7c-87bf-111e145d281b\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b \u003e GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x1fce6d014600\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110189966687473664","occurred_at":"2026-10-14T02:04:28Z","signature_id":"a63d52fcb7a18aef09e6286c67a8d5f921df82fc1ef1c88cc7261cf7af793684","pr_number":7510,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b3e3dde3ed6e86eca14ea52f55b0d41a64a9d455bb9f2e88e86dfa581de65743","failure_pattern":"failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T09:26:31Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/acrpull","occurrences":1}],"full_error_samples":["time=2026-10-12T10:17:07.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull description=\"Step acrpull\\n Kind: Helm\\n\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull values=\"map[image:mcr.microsoft.com/aks/msi-acrpull@sha256:b0cbba7044d5142f8545758e5a69b7b35e7f8c6af596bc972e164d2b345978a7 namespace:acrpull replicas:2 ttlRotationFraction:0.5]\"\ntime=2026-10-12T10:17:08.348Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.589Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull namespace=acrpull\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull dryRun=false\ntime=2026-10-12T10:17:08.838Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:17:08.976Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull\ntime=2026-10-12T10:22:16.134Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-10-12T10:22:16.135Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=acrpull err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/acrpull/acrpull not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7466/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109576437055885312","occurred_at":"2026-10-12T09:26:31Z","signature_id":"5ac3f50227d6a7076813d8bbbe9de0e8c8876a26e6800117bb2654c1677cfb10","pr_number":7466,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9edb2d39bdff9cf1586610e7e1778893f0414c6b0749314791e92df31b54ec6f","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T16:19:32Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","occurrences":1}],"full_error_samples":["time=2026-10-13T18:29:04.438Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-13T18:29:04.984Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:e73242a2-1d48-4914-b494-303251483672 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-ci01-maestro-j4029440.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j4029440] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]] zoneCount:3] fleet:map[namespace:fleet serviceAccount:fleet] image:map[digest:sha256:ccc6c4381bbbc6a7af65d4c3886e20baea76f7bb95ea6048b33ee31e8046e5cb registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:1176baba-6cca-46e9-86bb-3a9c587b6d0c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-10-13T18:29:04.984Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:29:05.371Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-10-13T18:29:05.621Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-10-13T18:29:05.621Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:29:05.751Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:39:13.113Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded\"\ntime=2026-10-13T18:39:13.113Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","signature_id":"2a551764d2de0ec2ce5493d79b0c6e2957d9ec9ebccce1e0772fad9dff29448f","pr_number":7502,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.6060606060606061,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-12T15:17:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x31481464c780\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vxxh5xhm4hm8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x314814c9ae20\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x31481464c780\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vxxh5xhm4hm8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-vxxh5xhm4hm8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x314814c9ae20\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vxxh5xhm4hm8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109664686109626368","occurred_at":"2026-10-12T15:17:08Z","signature_id":"bec2ded38a9a866918a5ba9ad1e9a064412687841aabfca8c59e91dc8567ad9c","pr_number":7477,"after_last_push_of_merged_pr":false}]}]},{"environment":"prod","summary":{"total_runs":19,"failed_runs":9,"raw_occurrences":18,"matched_occurrences":16,"runs_affected":7},"rows":[{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":9,"runs_affected":2,"failed_runs":2,"run_impact_percent":10.526315789473685,"anchor_occurrences":9,"after_last_push_count":55,"also_in":["STG"],"trend_counts":[4,5,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":21,"prior_last_seen_at":"2026-10-13T23:00:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x3d7b4ffcc3f0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2zdwd2kph86q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2zdwd2kph86q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: ed604105-da04-4fd7-8c78-5f275dcaf7e5\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x3d7b4ffcc3f0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2zdwd2kph86q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2zdwd2kph86q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n correlation ID: ed604105-da04-4fd7-8c78-5f275dcaf7e5\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x2bd9e6524b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-x9cvklpp6546/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-x9cvklpp6546/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 3ef2ff68-ea41-4d88-bf36-b2074aadb190\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x2bd9e6524b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-x9cvklpp6546/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-x9cvklpp6546/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: 3ef2ff68-ea41-4d88-bf36-b2074aadb190\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xeee231ed350\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sfwcv6bft6rc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sfwcv6bft6rc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 6cda8fb7-ec85-49b9-bf1a-d1253f1440fb\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xeee231ed350\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sfwcv6bft6rc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sfwcv6bft6rc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n correlation ID: 6cda8fb7-ec85-49b9-bf1a-d1253f1440fb\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110144270114492416","occurred_at":"2026-10-13T23:00:48Z","signature_id":"220b58b0981bf9fb42890cec7d6869d8135401c54c4baf28e36dc98264d4f358","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110144270114492416","occurred_at":"2026-10-13T23:00:48Z","signature_id":"6e98c6a8bad905db7b1a42a17c59f49e51d4d6e1972b5447f6fe20e2a5682000","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110144270114492416","occurred_at":"2026-10-13T23:00:48Z","signature_id":"72a6de2a87d243b3cb323bfdc7685d9fecf6b6339e5d0bc004ee291ab9e1a76b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110144270114492416","occurred_at":"2026-10-13T23:00:48Z","signature_id":"a8a8aa3d52b37f24913a0c9119ab31ad62ce98e3e471750d8e38c7ebd4b1573c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2110144270114492416","occurred_at":"2026-10-13T23:00:48Z","signature_id":"eb3558d5b7ef85d9c2d95b184af8b82db14d97bceaaa8a3bf0013490eebb95de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"53bef87c15633c3f568f024da6cc38f582808a0877b3c985db197193b820c49c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"6b9a8d8d2e8c40d51b4cce49a4ede0535c9a9742ac1cde3633d2dbd66229d867","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"816302c6f773d8a3809afa480580f0992dc6bf52d93be454eb54c90fc89aac18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"f724161a4b388d34a0d7848bd2cb5190dd4142eda26cf33576fb2df82c2e7a7c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T02:00:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x130faa8f320\u003e: \n failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bkm7lwv87ztr\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x130f942f8e0\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x130faa8f320\u003e: \n failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bkm7lwv87ztr\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bkm7lwv87ztr\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x130f942f8e0\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bkm7lwv87ztr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2110189368546168832","occurred_at":"2026-10-14T02:00:55Z","signature_id":"3e20acc848c2269da50d9c76e9d94a46ebbb03e0c5238076f79b8df8ef6b3923","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f4eb096a7612443e4e9b6d7bbb5b8a801d6e02c6b9b6e22b42ffb697b364a69c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T13:01:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-q7jjp7\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x298b49702ce0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-q7jjp7: failed waiting for cluster=\"cluster-zstream-4-21-q7jjp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-jd6h7xk8btv9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"name\": \"c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T14:02:09.836072198Z\",\n \"endTime\": \"2026-10-12T14:31:10.061357256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, cluster-network-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, openshift-route-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n correlation ID: 414509d7-91c9-4911-b025-fd41c0e067b1\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-q7jjp7\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x298b49702ce0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-q7jjp7: failed waiting for cluster=\"cluster-zstream-4-21-q7jjp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-jd6h7xk8btv9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"name\": \"c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T14:02:09.836072198Z\",\n \"endTime\": \"2026-10-12T14:31:10.061357256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, cluster-network-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-storage-operator, openshift-route-controller-manager, hosted-cluster-config-operator, redhat-operators-catalog, cluster-version-operator, redhat-marketplace-catalog, certified-operators-catalog, dns-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c104c66f-5dc3-4f93-b1cf-b5a10eca8fb8\n correlation ID: 414509d7-91c9-4911-b025-fd41c0e067b1\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109630653069791232","occurred_at":"2026-10-12T13:01:57Z","signature_id":"625fe3d4a7bcc05f82b6be9c1b459cb3509760d899d8c1e3dcb5cf92e4d63ae3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-12T06:42:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ptqdgd\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x21d67b55ea20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-ptqdgd: failed waiting for cluster=\"cp-ystream-upgrade-5-0-ptqdgd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dznstqd64m54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"name\": \"9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T07:51:24.545231929Z\",\n \"endTime\": \"2026-10-12T08:20:29.935309867Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n correlation ID: 7a60ad0c-37f6-45ef-98dd-efdfad687d6c\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-ptqdgd\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x21d67b55ea20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-ptqdgd: failed waiting for cluster=\"cp-ystream-upgrade-5-0-ptqdgd\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dznstqd64m54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"name\": \"9d14066e-1629-48f8-aa68-f556c4fd7044\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T07:51:24.545231929Z\",\n \"endTime\": \"2026-10-12T08:20:29.935309867Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/9d14066e-1629-48f8-aa68-f556c4fd7044\n correlation ID: 7a60ad0c-37f6-45ef-98dd-efdfad687d6c\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109535177171537920","occurred_at":"2026-10-12T06:42:34Z","signature_id":"fd4fc621d01f385247f810320f7e86a4ebf9edde1b95ddd444f5031ee94b4826","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T00:05:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x151c97ead140\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-p86tvztrm7hd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"name\": \"01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T01:36:23.272806003Z\",\n \"endTime\": \"2026-10-12T02:05:32.513113167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n correlation ID: e3c8027d-936a-4cfa-802c-eac5497d3369\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x151c97ead140\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-p86tvztrm7hd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"name\": \"01aaf869-754a-4412-a8d0-48b576bc7f7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T01:36:23.272806003Z\",\n \"endTime\": \"2026-10-12T02:05:32.513113167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-p86tvztrm7hd--managed/providers/Microsoft.Compute/virtualMachines/u6e7j5d8a2b3s1a-cordon-np-bp4qm-n5n47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01aaf869-754a-4412-a8d0-48b576bc7f7b\n correlation ID: e3c8027d-936a-4cfa-802c-eac5497d3369\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","occurred_at":"2026-10-12T00:05:13Z","signature_id":"a33be23ec9f3ec13df877ffe26bd388352f59b753cee7526ea22101141d231fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"87227a922c90e8b74b9cd5f3e1315585fd49c7030b59796956bc7a208ed9e9d1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T00:05:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x38da2adfeb00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-kbw6v2d2bhz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\",\n \"name\": \"4899c274-4471-4ab9-871b-22368aa84095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:35:15.167750325Z\",\n \"endTime\": \"2026-10-12T01:04:20.403308122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-fk5wl-zdgdp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-fk5wl-zdgdp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n correlation ID: 79b16636-2655-4af8-9ac0-f8e4eda0ad35\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x38da2adfeb00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-kbw6v2d2bhz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\",\n \"name\": \"4899c274-4471-4ab9-871b-22368aa84095\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T00:35:15.167750325Z\",\n \"endTime\": \"2026-10-12T01:04:20.403308122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-fk5wl-zdgdp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-fk5wl-zdgdp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kbw6v2d2bhz5--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-fk5wl-zdgdp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4899c274-4471-4ab9-871b-22368aa84095\n correlation ID: 79b16636-2655-4af8-9ac0-f8e4eda0ad35\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109435185735929856","occurred_at":"2026-10-12T00:05:13Z","signature_id":"ca3816013b61a6551e43b1071dcaa2fa8f2e2f088ce6dca69491ec625e587d83","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8effde2aab4f5beaedda3d8b16dda29c381b8de0bb2c433491a2eb475fa67fb2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T00:58:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc79feec4060\u003e: \n resource group \"np-autoscaling-xtg8j7kg4fwz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xtg8j7kg4fwz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"name\": \"6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T02:47:19.509292457Z\",\n \"endTime\": \"2026-10-12T03:11:19.252244159Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc79feec4060\u003e: \n resource group \"np-autoscaling-xtg8j7kg4fwz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xtg8j7kg4fwz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"name\": \"6af3ef7c-4974-4b33-bebb-ed53f86cb854\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T02:47:19.509292457Z\",\n \"endTime\": \"2026-10-12T03:11:19.252244159Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2109448556992532480","occurred_at":"2026-10-12T00:58:22Z","signature_id":"92ca5810cc30b4aac8e0fae2571490fb634c25c0c1d6fd8a60f745c71eb4e4ce","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.2631578947368425,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["STG"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-12T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:175]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n \u003c*errors.joinError | 0x2d289f100030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d289e7b3500\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2d289e7b34e0\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2d289f0dae60\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x2d289e6a6040\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2d289e6a6020\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2d289e168dc0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:175]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n \u003c*errors.joinError | 0x2d289f100030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d289e7b3500\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2d289e7b34e0\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2d289f0dae60\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x2d289e6a6040\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2d289e6a6020\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2d289e168dc0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2109781178750865408","occurred_at":"2026-10-12T23:00:05Z","signature_id":"74819dbc97c4d84b5230e834e8fd3d8a06bb306d8df4e0b9e906ef2406d3f4fa","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":54,"matched_occurrences":54,"runs_affected":4},"rows":[{"environment":"stg","failure_pattern_id":"d523a0d2817fdbba88731dd1892c07aa03075f35ab21021c384553c8aa80da9e","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":11,"after_last_push_count":11,"trend_counts":[0,11,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T00:11:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hkwrn8b75t7c/cluster-candidate-4-21-8m4xcg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15168f06cae0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-8m4xcg: failed waiting for cluster=\"cluster-candidate-4-21-8m4xcg\" in resourcegroup=\"rg-candidate-4-21-hkwrn8b75t7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8479abf5-39da-412b-bb04-87374bf3b757\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8479abf5-39da-412b-bb04-87374bf3b757\",\n \"name\": \"8479abf5-39da-412b-bb04-87374bf3b757\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T00:45:19.685007768Z\",\n \"endTime\": \"2026-10-13T01:14:27.351105107Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8479abf5-39da-412b-bb04-87374bf3b757\n correlation ID: 46a9a6c6-2ede-454b-9940-a881ca6f8b60\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hkwrn8b75t7c/cluster-candidate-4-21-8m4xcg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15168f06cae0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-8m4xcg: failed waiting for cluster=\"cluster-candidate-4-21-8m4xcg\" in resourcegroup=\"rg-candidate-4-21-hkwrn8b75t7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8479abf5-39da-412b-bb04-87374bf3b757\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8479abf5-39da-412b-bb04-87374bf3b757\",\n \"name\": \"8479abf5-39da-412b-bb04-87374bf3b757\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T00:45:19.685007768Z\",\n \"endTime\": \"2026-10-13T01:14:27.351105107Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8479abf5-39da-412b-bb04-87374bf3b757\n correlation ID: 46a9a6c6-2ede-454b-9940-a881ca6f8b60\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3152b3c1a500\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rtr2wnffp4rh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db0a48c2-f9bb-4a38-94f5-afccbc7d779c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db0a48c2-f9bb-4a38-94f5-afccbc7d779c\",\n \"name\": \"db0a48c2-f9bb-4a38-94f5-afccbc7d779c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T00:45:30.036006045Z\",\n \"endTime\": \"2026-10-13T01:14:37.44566298Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db0a48c2-f9bb-4a38-94f5-afccbc7d779c\n correlation ID: 6b6c3060-a1bb-4638-afa6-7fb92caae9a8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3152b3c1a500\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rtr2wnffp4rh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db0a48c2-f9bb-4a38-94f5-afccbc7d779c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db0a48c2-f9bb-4a38-94f5-afccbc7d779c\",\n \"name\": \"db0a48c2-f9bb-4a38-94f5-afccbc7d779c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T00:45:30.036006045Z\",\n \"endTime\": \"2026-10-13T01:14:37.44566298Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db0a48c2-f9bb-4a38-94f5-afccbc7d779c\n correlation ID: 6b6c3060-a1bb-4638-afa6-7fb92caae9a8\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x15a1e46902c0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-xgppw4wmq2g9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f006a7a-53bf-4566-93d0-8202c11da346\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f006a7a-53bf-4566-93d0-8202c11da346\",\n \"name\": \"8f006a7a-53bf-4566-93d0-8202c11da346\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T00:52:36.6753625Z\",\n \"endTime\": \"2026-10-13T01:21:37.350837076Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f006a7a-53bf-4566-93d0-8202c11da346\n correlation ID: 8ed813fc-cdf9-4562-abe6-9eb495caf777\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x15a1e46902c0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-xgppw4wmq2g9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f006a7a-53bf-4566-93d0-8202c11da346\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f006a7a-53bf-4566-93d0-8202c11da346\",\n \"name\": \"8f006a7a-53bf-4566-93d0-8202c11da346\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T00:52:36.6753625Z\",\n \"endTime\": \"2026-10-13T01:21:37.350837076Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f006a7a-53bf-4566-93d0-8202c11da346\n correlation ID: 8ed813fc-cdf9-4562-abe6-9eb495caf777\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"146b78c30965fd0eb4cbd4f3ce0bd0c961d9198f5c4dc40c96ac1593fd4e4ab9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"211d9e86a008ecd6fa6d766c44aaa6f554e1f156d94e1149f30796441f6415ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"3b5ed882f5b8a5dee147e684db818ac7687a223531ae10d7c191e78de9b6c5d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"40f1808c5e69a39d8919133ff46ba50f8fb2c13cb58040a5c016fd0004ac9887","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"59d2a60065096b9be506e383118f18e05b638cdb65d7b202ec70d69d4f59b355","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"989ac49898734cb91e97e4c2757ae452dc10467a4f141a00c0154a27c5b72fba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"a0b1a2bcc812987c03bd737995dbe8a3392c8e513bd6bf93c14955431fbcf530","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"c554889764dcd866f33751f9ea3ec152dc93da7a1970f0d1cd05c52a09638a14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"d6e70897832ff697a58dcb131c49942425a4086bb43ceb0ccd39840ef963d9ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"de03542f2c0a500421c1c841783f10763335393087d269d92f9143863bfae364","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"f2c80a85c8e7b37551fe65b59ebb035bbcafb800625336241b1bdba175d2c3c5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":10,"after_last_push_count":16,"trend_counts":[0,10,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T00:11:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x1854e4898c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-v622rq7cjdv8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-v622rq7cjdv8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: e25d2783-3afe-4763-a826-691a13ceb4bb\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x1854e4898c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-v622rq7cjdv8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-v622rq7cjdv8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: e25d2783-3afe-4763-a826-691a13ceb4bb\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-m6j7kp\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x13a151ae2450\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-lnhr67n47jrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-m6j7kp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-lnhr67n47jrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-m6j7kp\n correlation ID: b7196ccc-8430-45a5-9628-149904d927cb\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-m6j7kp\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x13a151ae2450\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-lnhr67n47jrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-m6j7kp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-lnhr67n47jrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-m6j7kp\n correlation ID: b7196ccc-8430-45a5-9628-149904d927cb\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x1d06db38a6c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-kbw2fdfhjc7f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-kbw2fdfhjc7f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 581e9f52-3486-4da8-af1a-3a90318bf15f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x1d06db38a6c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-kbw2fdfhjc7f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-kbw2fdfhjc7f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 581e9f52-3486-4da8-af1a-3a90318bf15f\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"0530912e63ebaa786157b9be3e51c21406424ce71acb1245df7befa69a986861","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"1f8bf6e9c71e6e658f0eb525d46b51b68287c4e5c7aaddcbb7cef978804c7940","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"4105e5196c9b06f18563004fc624e7fdff756bc7f4ad9f2dda70491caf4acdeb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"48ac4f417bf672de53bb364e3abbcb506f41e3034429b45718a217be8d500529","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"4c78b812f71cdd2a83ca79b888006363af10ddc53913faa2c746dedc64e788a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"5b2aa9ee93a950d280f847817e3d121f5f23abfcc2b736b952115378264799b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"6167d4d2748153d4edd45cc896d5128e080cb0b4489ac3e36ce7b90d7193d69d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"631420272f54b0b3c643878049b1616ea34df024e3749006944c970a346ba9e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"8d4b8db6d1607e9537a81149ac935a37bf84f1d67a309eb989f6dd18330bf924","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"9231c7ee3132e8b1b1306f36d044cae9277c3ecd3a95c885de54a0d846045ce5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":9,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":9,"after_last_push_count":61,"also_in":["PROD"],"trend_counts":[4,5,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-13T23:00:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3bf246e81b90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-kdzwfbjg55gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-kdzwfbjg55gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: c883dc6d-a815-4804-970f-295c17dcee98\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3bf246e81b90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-kdzwfbjg55gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-kdzwfbjg55gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n correlation ID: c883dc6d-a815-4804-970f-295c17dcee98\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x1fbd9ed3dd40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-69wnspjznlc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-69wnspjznlc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 3c34a173-f084-4c74-8bf8-8d8da2ea7e20\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x1fbd9ed3dd40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-69wnspjznlc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-69wnspjznlc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n correlation ID: 3c34a173-f084-4c74-8bf8-8d8da2ea7e20\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x25636a4bb950\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nh5qcpqmqsm8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nh5qcpqmqsm8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: eb7e7c6f-c7e9-46a0-8d7d-c420ec91eb1d\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x25636a4bb950\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nh5qcpqmqsm8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nh5qcpqmqsm8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n correlation ID: eb7e7c6f-c7e9-46a0-8d7d-c420ec91eb1d\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110144221439594496","occurred_at":"2026-10-13T23:00:48Z","signature_id":"0f887016559b59b9bbf26868ef6fd6c97974eb6c58fbcc78f8419c2b124c14e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110144221439594496","occurred_at":"2026-10-13T23:00:48Z","signature_id":"53c0722a3f14666623651b51490ed67e6a03fae314ab56917e282a5cfb11ecaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110144221439594496","occurred_at":"2026-10-13T23:00:48Z","signature_id":"79753bdaa9c52ea30e2277726adde7adffc2cbab5469de8409cd0984ea857170","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110144221439594496","occurred_at":"2026-10-13T23:00:48Z","signature_id":"8ed204b2f8e9d46fb245cbb52c9c47dbd075af4a7d556c3e77c721289a4a2141","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110144221439594496","occurred_at":"2026-10-13T23:00:48Z","signature_id":"f201c230675546b19aed1786663a5ad0dde13107c5fa4f656b0e72537479e774","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"2e2ec56fa735e3c6fb65ee60df50e7e7dace6166364141c3f1bfe0771ff4e918","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"344ee26cca90a92005b8b763492dab51ac06c3fcc9e83e7f0eae01cb2e7c5117","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"be78c1649579da41709f8c31cbe7770f3c078a5835b2696b6e43cfb6b4a7b282","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"ea927c68dc1333367697e506d3d20b5dcec61691018b2f2c2860e0c4025a33af","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":9,"after_last_push_count":11,"trend_counts":[0,9,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-13T00:11:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x31faef804060\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-w5jlcn\" resourcegroup=\"rg-np-version-upgrade-w5jlcn-chd7h2jfcvzd\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-w5jlcn-chd7h2jfcvzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-w5jlcn/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-w5jlcn-chd7h2jfcvzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-w5jlcn/nodePools/npupgrade-4-20\n correlation ID: 6037c313-80d6-4b5c-af77-583498b149c9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x31faef804060\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-w5jlcn\" resourcegroup=\"rg-np-version-upgrade-w5jlcn-chd7h2jfcvzd\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-w5jlcn-chd7h2jfcvzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-w5jlcn/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-w5jlcn-chd7h2jfcvzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-w5jlcn/nodePools/npupgrade-4-20\n correlation ID: 6037c313-80d6-4b5c-af77-583498b149c9\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x258acfd8b940\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-zfs5gc\" resourcegroup=\"rg-np-version-upgrade-zfs5gc-ksr8l4d4q4gw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zfs5gc-ksr8l4d4q4gw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-zfs5gc/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zfs5gc-ksr8l4d4q4gw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-zfs5gc/nodePools/npupgrade-4-21\n correlation ID: 0b2d3160-3474-40bc-a9f5-2792d7568074\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x258acfd8b940\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-zfs5gc\" resourcegroup=\"rg-np-version-upgrade-zfs5gc-ksr8l4d4q4gw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zfs5gc-ksr8l4d4q4gw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-zfs5gc/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zfs5gc-ksr8l4d4q4gw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-zfs5gc/nodePools/npupgrade-4-21\n correlation ID: 0b2d3160-3474-40bc-a9f5-2792d7568074\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b553dfaa40\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-v69sgj\" resourcegroup=\"rg-np-dg-minor-v69sgj-tl6w2zktdhh2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-v69sgj-tl6w2zktdhh2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-v69sgj/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-v69sgj-tl6w2zktdhh2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-v69sgj/nodePools/npdg-422-420\n correlation ID: dfeeac19-d930-4394-8440-1041d3f8556f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b553dfaa40\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-v69sgj\" resourcegroup=\"rg-np-dg-minor-v69sgj-tl6w2zktdhh2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-v69sgj-tl6w2zktdhh2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-v69sgj/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-v69sgj-tl6w2zktdhh2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-v69sgj/nodePools/npdg-422-420\n correlation ID: dfeeac19-d930-4394-8440-1041d3f8556f\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"4e8db88c56f6a1c6f912e13ae0d689f1a333de5aa3744afd9ee5e3288b7bd79e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"5510045c8ae671875a8bd64a610550633c9c7fed24b71dcd19555786dc397913","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"b233cf2c244df8c1904bebd0a53851c25328d578dd76169cde0325b0795a4f65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"b3ec023fa2707218bf83b6231554dfa12c6c1cff497571ffd2e1842c36ab2ba1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"b5d353115c29b3e6152cbe4210dd370af488b760fa291c5307df1913e9ff5125","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"c204676ef8585938e7d8b097dc91ec23f082daf972f23c3e551ea7630339f76c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"e3b620620ea20cbc1096b2ec20b6fbd7879e0dbfb86875c0561406f388ffca54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"e9853471d845a3beaad9fbe3165e0eb0cb66453e8133db5a97bf0006f63b300f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"f63ed555dda3b7d4ed9c19ea736208f188370278b4e5347c15eaa529fa456991","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"53cdeeaf6be0a342819bcbeaa249de72112012b896dbb65176b19f6c0f1a6d61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[5,1,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-13T23:00:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-kxcsmnps9zfz/cluster-nightly-4-21-j9wst5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x21388a648d60\u003e: \n failed to create HCP cluster cluster-nightly-4-21-j9wst5: failed waiting for cluster=\"cluster-nightly-4-21-j9wst5\" in resourcegroup=\"rg-nightly-4-21-kxcsmnps9zfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6943378a-a2ac-4799-849e-25f8e97d936d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6943378a-a2ac-4799-849e-25f8e97d936d\",\n \"name\": \"6943378a-a2ac-4799-849e-25f8e97d936d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T23:24:54.16266447Z\",\n \"endTime\": \"2026-10-13T23:54:01.266968226Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6943378a-a2ac-4799-849e-25f8e97d936d\n correlation ID: 11b5d2e6-bd60-46f6-9cea-eb83bb792173\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-kxcsmnps9zfz/cluster-nightly-4-21-j9wst5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x21388a648d60\u003e: \n failed to create HCP cluster cluster-nightly-4-21-j9wst5: failed waiting for cluster=\"cluster-nightly-4-21-j9wst5\" in resourcegroup=\"rg-nightly-4-21-kxcsmnps9zfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6943378a-a2ac-4799-849e-25f8e97d936d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6943378a-a2ac-4799-849e-25f8e97d936d\",\n \"name\": \"6943378a-a2ac-4799-849e-25f8e97d936d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T23:24:54.16266447Z\",\n \"endTime\": \"2026-10-13T23:54:01.266968226Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6943378a-a2ac-4799-849e-25f8e97d936d\n correlation ID: 11b5d2e6-bd60-46f6-9cea-eb83bb792173\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-mjltzvcvvmbh/cluster-nightly-4-22-xv7w42 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xe85a0514ac0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-xv7w42: failed waiting for cluster=\"cluster-nightly-4-22-xv7w42\" in resourcegroup=\"rg-nightly-4-22-mjltzvcvvmbh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n \"name\": \"df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:26:58.898405913Z\",\n \"endTime\": \"2026-10-12T23:56:08.010677982Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n correlation ID: 99498835-8536-458a-9168-4524819cc6bd\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-mjltzvcvvmbh/cluster-nightly-4-22-xv7w42 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xe85a0514ac0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-xv7w42: failed waiting for cluster=\"cluster-nightly-4-22-xv7w42\" in resourcegroup=\"rg-nightly-4-22-mjltzvcvvmbh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n \"name\": \"df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:26:58.898405913Z\",\n \"endTime\": \"2026-10-12T23:56:08.010677982Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df7b36b8-c15c-41a8-87f0-fb3ee1d8a874\n correlation ID: 99498835-8536-458a-9168-4524819cc6bd\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-lmbgqw\" at install version 4.22.0-0.nightly-multi-2026-10-12-042614\nUnexpected error:\n \u003c*fmt.wrapError | 0x2912ce16d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-lmbgqw: failed waiting for cluster=\"cp-ystream-upgrade-4-23-lmbgqw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xm2k5t9rnhbb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n \"name\": \"40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:30:45.035752487Z\",\n \"endTime\": \"2026-10-12T23:59:45.878026173Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n correlation ID: 61a61582-38bf-4dc1-a572-663613c43a27\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-lmbgqw\" at install version 4.22.0-0.nightly-multi-2026-10-12-042614\nUnexpected error:\n \u003c*fmt.wrapError | 0x2912ce16d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-lmbgqw: failed waiting for cluster=\"cp-ystream-upgrade-4-23-lmbgqw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xm2k5t9rnhbb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n \"name\": \"40887f6f-9dfa-4fd4-a671-e1259364b51d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:30:45.035752487Z\",\n \"endTime\": \"2026-10-12T23:59:45.878026173Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/40887f6f-9dfa-4fd4-a671-e1259364b51d\n correlation ID: 61a61582-38bf-4dc1-a572-663613c43a27\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2110144221439594496","occurred_at":"2026-10-13T23:00:48Z","signature_id":"44b8c47df3900af37c7dea6efa7b21475eabaf8a68e245fae0991db0fec71f51","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"32cce48c70cd78aaab0ca1cf29578d7e7f790aac0f49595cdef30c5b9fbbae37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"5b3bd45c8acbe1b8c11b2ffcb06fab0944c8108dcebe086fa39dfd4f4016f6f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"93d83589a75ababb8b2f24abad716697f692af996c7f0f1a8f51065a65539f5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"b867162c92e79c7b8c388045402cfc0fe873015bb7e68fb7d92ed60c1269d3aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"c15fdeb0bd481599ce3a3469d527a846870c693fc60106f4ccf60e4d23ab0f56","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3a91d2342aecbce462c7c3203c6ddaa1ebb0f773083d4b54f1ef0a3e6486ca04","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,3,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-13T00:11:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-xw6srx\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x2299302636e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-5tqz59mk2x4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-xw6srx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-5tqz59mk2x4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-xw6srx\n correlation ID: d5115cbd-66f0-411a-9ed8-23fc462b21fe\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-xw6srx\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x2299302636e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-5tqz59mk2x4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-xw6srx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-5tqz59mk2x4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-xw6srx\n correlation ID: d5115cbd-66f0-411a-9ed8-23fc462b21fe\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-xd29vz\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x33ccd41339b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7gnkhhshzxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xd29vz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7gnkhhshzxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xd29vz\n correlation ID: 1a4d52b7-b1bd-42a6-8e35-cd6a15284e2b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-xd29vz\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x33ccd41339b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7gnkhhshzxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xd29vz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7gnkhhshzxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xd29vz\n correlation ID: 1a4d52b7-b1bd-42a6-8e35-cd6a15284e2b\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-gk5dl2\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x1206c2abaf90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qct9sm5zfdhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gk5dl2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qct9sm5zfdhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gk5dl2\n correlation ID: 72fd87ea-2d1e-422a-b1c8-66816eaede96\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-gk5dl2\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x1206c2abaf90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qct9sm5zfdhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gk5dl2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qct9sm5zfdhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gk5dl2\n correlation ID: 72fd87ea-2d1e-422a-b1c8-66816eaede96\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"3a0f2456b41051fa04e3a62584ed9deef034a73352e325d422d69eac87de652a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"6643ddc260175fbcc3e51729ad7befc0e16ab3484b1b2bfee84e6ab0622da395","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2109799288455303168","occurred_at":"2026-10-13T00:11:54Z","signature_id":"c870c0607c182e7083d06fe1b8dbaa6ee6caac0f1a1a981bae76d454306ef904","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"38696c55a4f3351eeef461deb7069c172bd5ca5ddee5655459be99523f263138","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV","PROD"],"trend_counts":[1,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-14T02:00:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e2df3fce3a0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-cxh9xv97rj9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\",\n \"name\": \"0195b61d-2798-45a3-a546-b2e316a8e4be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:10:09.167253269Z\",\n \"endTime\": \"2026-10-14T03:39:12.28722327Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n correlation ID: 6295da04-1f90-4641-a016-0502957f2d44\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e2df3fce3a0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-cxh9xv97rj9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\",\n \"name\": \"0195b61d-2798-45a3-a546-b2e316a8e4be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-14T03:10:09.167253269Z\",\n \"endTime\": \"2026-10-14T03:39:12.28722327Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-cxh9xv97rj9f--managed/providers/Microsoft.Compute/virtualMachines/v9u7s9d4e2s3d9f-cordon-np-j7dj6-snwxb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0195b61d-2798-45a3-a546-b2e316a8e4be\n correlation ID: 6295da04-1f90-4641-a016-0502957f2d44\n client request ID: unavailable\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x371cbc791620\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-n9bmt8cwvq6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\",\n \"name\": \"7da88189-d1ef-4299-b869-95f7299be069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:47:30.718672913Z\",\n \"endTime\": \"2026-10-13T00:16:35.706367535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n correlation ID: f89db514-fba8-4f47-b43b-0ea8cd9a0b5e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x371cbc791620\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-n9bmt8cwvq6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\",\n \"name\": \"7da88189-d1ef-4299-b869-95f7299be069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:47:30.718672913Z\",\n \"endTime\": \"2026-10-13T00:16:35.706367535Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-n9bmt8cwvq6g--managed/providers/Microsoft.Compute/virtualMachines/c1m4z2y4h9h6m1o-cordon-np-qfzbb-vrnqt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da88189-d1ef-4299-b869-95f7299be069\n correlation ID: f89db514-fba8-4f47-b43b-0ea8cd9a0b5e\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2110189439996137472","occurred_at":"2026-10-14T02:00:55Z","signature_id":"fb5045f1b194ee1b057ae791bec58fa7b505b98c7c93aa828d301e5d4be214e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"69b2e09a428ec7ca6e59b6def93f3906763678b69f1bb0465068db0038f587d3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"34aecee469de1d884ea46e23ff6727d6f9c494870f6a79f955e34ff82db132ab","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-n6c4c4fd2ssk\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-14T02:00:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x95f8488fc80\u003e: \n failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-n6c4c4fd2ssk\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-n6c4c4fd2ssk\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x95f84458c30\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:262]: failed to apply external auth config with prefix policy None\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x95f8488fc80\u003e: \n failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-n6c4c4fd2ssk\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-n6c4c4fd2ssk\\\" for cluster=\\\"ea-cluster\\\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x95f84458c30\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-n6c4c4fd2ssk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2110189439996137472","occurred_at":"2026-10-14T02:00:55Z","signature_id":"9a01b0129009725c13101877a59ee924feac36de5d9367732d0e5b62c59478f5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a3c74e7b398e04fb715ace48f34fd905bc9ad59fa3d32a732969fe4992fdd590","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, konnectivity-agent, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-4c2bdskg9txb/cluster-nightly-5-0-44gd6d should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc4ca110a7e0\u003e: \n failed to create HCP cluster cluster-nightly-5-0-44gd6d: failed waiting for cluster=\"cluster-nightly-5-0-44gd6d\" in resourcegroup=\"rg-nightly-5-0-4c2bdskg9txb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n \"name\": \"bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:27:12.849511271Z\",\n \"endTime\": \"2026-10-12T23:56:15.713757928Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, ingress-operator, konnectivity-agent, cluster-version-operator, cluster-storage-operator, community-operators-catalog, olm-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n correlation ID: 37b4d83d-eab7-42e6-8d94-8b862a090192\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-4c2bdskg9txb/cluster-nightly-5-0-44gd6d should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc4ca110a7e0\u003e: \n failed to create HCP cluster cluster-nightly-5-0-44gd6d: failed waiting for cluster=\"cluster-nightly-5-0-44gd6d\" in resourcegroup=\"rg-nightly-5-0-4c2bdskg9txb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n \"name\": \"bf9bc1de-04dc-49de-bcdd-655adcca76f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:27:12.849511271Z\",\n \"endTime\": \"2026-10-12T23:56:15.713757928Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-network-operator, ingress-operator, konnectivity-agent, cluster-version-operator, cluster-storage-operator, community-operators-catalog, olm-operator, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf9bc1de-04dc-49de-bcdd-655adcca76f1\n correlation ID: 37b4d83d-eab7-42e6-8d94-8b862a090192\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"99165d131f611e1774b5c8402a98875009d3632ad9f1c583245d77f5a5a09a7d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"433ac3e41264da043a734cacbf5bc6523c2571e994c16b98d54f5b82c0cc686a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-12T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-23-jglxt7lcshss/cluster-nightly-4-23-r26z2q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d73d9d10e00\u003e: \n failed to create HCP cluster cluster-nightly-4-23-r26z2q: failed waiting for cluster=\"cluster-nightly-4-23-r26z2q\" in resourcegroup=\"rg-nightly-4-23-jglxt7lcshss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\",\n \"name\": \"41bf70ba-7344-4c62-9364-b6165697473d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:27:02.793421826Z\",\n \"endTime\": \"2026-10-12T23:56:08.063137348Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-node-tuning-operator, dns-operator, machine-approver, cluster-storage-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-version-operator, ingress-operator, control-plane-pki-operator, olm-operator, cluster-network-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n correlation ID: bbc14ae0-9ae0-4116-9b30-00ddeee9cda8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-23-jglxt7lcshss/cluster-nightly-4-23-r26z2q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d73d9d10e00\u003e: \n failed to create HCP cluster cluster-nightly-4-23-r26z2q: failed waiting for cluster=\"cluster-nightly-4-23-r26z2q\" in resourcegroup=\"rg-nightly-4-23-jglxt7lcshss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\",\n \"name\": \"41bf70ba-7344-4c62-9364-b6165697473d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-12T23:27:02.793421826Z\",\n \"endTime\": \"2026-10-12T23:56:08.063137348Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, cluster-node-tuning-operator, dns-operator, machine-approver, cluster-storage-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, packageserver, cluster-version-operator, ingress-operator, control-plane-pki-operator, olm-operator, cluster-network-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/41bf70ba-7344-4c62-9364-b6165697473d\n correlation ID: bbc14ae0-9ae0-4116-9b30-00ddeee9cda8\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"e1a8bd9667a06efef689c609b6312d7ffbd0831f396ff2202ea50c17e42be65b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96eb8165e1517c94f94344a079943853da38a18f87e198ac97e2ca5f84f1f2e3","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0],"trend_range":"2026-10-12..2026-10-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":1,"prior_last_seen_at":"2026-10-12T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n \u003c*errors.joinError | 0x3d800eb17200\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x3d800f297260\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x3d800f297240\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x3d800e191e00\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x3d800e204000\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x3d800eb05ee0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x3d800e0fab40\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n \u003c*errors.joinError | 0x3d800eb17200\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x3d800f297260\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x3d800f297240\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x3d800e191e00\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x3d800e204000\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x3d800eb05ee0\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x3d800e0fab40\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2109781178767642624","occurred_at":"2026-10-12T23:00:05Z","signature_id":"bcf5d924fc188320a62950d409f473b61ec182e32796d6b849eb900353c06dc6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}