{"meta":{"start_date":"2026-10-07","end_date":"2026-10-07","start_at":"2026-10-07T00:00:00Z","end_at":"2026-10-08T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T07:03:39Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":71,"failed_runs":61,"raw_occurrences":431,"matched_occurrences":425,"runs_affected":55},"rows":[{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":135,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":135,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,135],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-30T06:47:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2f8378088558\u003e: \n    resource group \"kusto-logs-zdcz6ngd7l8s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zdcz6ngd7l8s\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"name\": \"ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T13:55:49.637158017Z\",\n      \"endTime\": \"2026-10-07T14:19:49.807578144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vu3tto21eb5vs2oeuv4d035r4kh9q still exists (deletion dispatched at 2026-10-07T13:56:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2f8378088558\u003e: \n    resource group \"kusto-logs-zdcz6ngd7l8s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zdcz6ngd7l8s\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"name\": \"ca8af611-54d1-469b-80b1-afacd1e9f0c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T13:55:49.637158017Z\",\n      \"endTime\": \"2026-10-07T14:19:49.807578144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vu3tto21eb5vs2oeuv4d035r4kh9q still exists (deletion dispatched at 2026-10-07T13:56:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xf860122e060\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d7rnngth6rmt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s5wrnm\" in resourcegroup=\"rg-zstream-upgrade-4-21-d7rnngth6rmt\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"name\": \"de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:14:15.375309734Z\",\n      \"endTime\": \"2026-10-07T14:38:20.14126941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtq0dj9uuqrc354smigdjo5gckvsa still exists (deletion dispatched at 2026-10-07T14:14:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xf860122e060\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d7rnngth6rmt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s5wrnm\" in resourcegroup=\"rg-zstream-upgrade-4-21-d7rnngth6rmt\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"name\": \"de2aa0e7-6bfa-4ba6-91bf-20123b9953f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:14:15.375309734Z\",\n      \"endTime\": \"2026-10-07T14:38:20.14126941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtq0dj9uuqrc354smigdjo5gckvsa still exists (deletion dispatched at 2026-10-07T14:14:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3d715b6dc0c0\u003e: \n    resource group \"admin-api-serialconsole-kgtb4656t7wx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-kgtb4656t7wx\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"name\": \"63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:05:26.627856798Z\",\n      \"endTime\": \"2026-10-07T14:29:30.238184303Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtu5p75nk86f54m2uds6vreosd2p2 still exists (deletion dispatched at 2026-10-07T14:05:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3d715b6dc0c0\u003e: \n    resource group \"admin-api-serialconsole-kgtb4656t7wx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-kgtb4656t7wx\" to finish deleting: GET https://rp.j9974528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"name\": \"63b430ac-4aff-4ba1-9a60-4787a227b94b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:05:26.627856798Z\",\n      \"endTime\": \"2026-10-07T14:29:30.238184303Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9vtu5p75nk86f54m2uds6vreosd2p2 still exists (deletion dispatched at 2026-10-07T14:05:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"00770c4fe97dfc58f45740186e91ff796f07117f2e217de3f77804d73f958c0a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"013b36a17be5e1755039e5e47a5699ae4977c8c5bad7e790f67e10084353346e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"029c4854e61d0d203eced163c848473e3858d9fb8f5715fbb1ab826a86732057","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"02b669fab2f3a752654dc9a34f097918516a7c5968bad6ff55e521951f3af1a4","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"070933d7361dc14a21e4c08bdac9bd6cd2038d91232c2cc7958eb579eb093158","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"0e6e93a01ccbe18930c88f6800d0218371948c8b0fcbf57358cce8f18990f3a1","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"0e7b9a1304f6b5950ff5a6780c8569a77bfb9dd61b6ace6197c47db81e546ab5","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"0f75db649ea9fa6db01a02d3b5637d779ce9e29da1a572be08081e8d2fb66e8a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"14983add3cbb52204586976e52460dc077bc8f9f1bba2975dd821355d1384070","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"1bc6da96879e0d6f1e2c7c45a23431c8c4f6454dec82e919b9545e6949a18f21","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"1be34856b46f8d6a9c44093844c8700f3269be9e60a78622e9533e8933047309","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"21c48ae3defdd43c7577c013e6f6c9fdd335dbfe0b89683654111c1b5ed39e86","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"26681a4be74afffb2e578b6b4027025365128e4c366f444bc762944b6c1b93df","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"276eea38d768faf75d57e95a2137268e1a8a0f332f48a71f2d800f4634e514fd","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"285faa2939852d449718aa4c429df428b96828f95653c2e3cb13ef94e108c9b6","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"286cafcce7f2030ffc52038d15d989fbe508aa9222743b5b8ab4a6c9345e520b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"294252fbbb161d3da22685c5b51e11a3f5e1fab801d5d967f03d3d6ddddbd6a0","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"2cd950718431e65c60c1449982bcf45b52fc9aa594f2e7f4733dab853ecd4237","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"3435cef6023d2bba4e541a2c09290178c6c9c1e592a41ec1fc2cbe0afc333879","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"3ed6e29ae24b22e936c139bddd29259000074bf38703eb0fdb9831e5f87da761","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"45fa548a5e7b5c65345ec02c8e2ab623bbe96ea387f003a14bf0fe82619d5276","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"48e55fcbb84d11612bdbc4e6665d9c301500141b07dfb62e29d7676b97884aca","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4af3a70cd20df1fb98271c331f06b303636ded7bdbad6e22483d028f110d6c72","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4b9bc53d8926afbba2ea2d19edb4820af2e72e4808254b23e4b76a910553f9c6","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4d27311f1e6a388347006b63a44e786c6632627915c8eb4e51a9fd9526df314b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5575d92252fa76c383dbc5dd7c06851a1356391e4e7aa3ca4aa803c9d3683f26","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"560f0c3953616fa100db0341bb4bdbea0d8e6993955774de086a0dcc2ed1e90e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5743dbe7878ad72b0dfce27b3e213f15fac28467e47138f4c8afcf36a31c1ffb","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5855b5533b4e07a4cb53c7524cbdc5aa4a741605c71ee80ff83ba4cbd790a596","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"5ff85e07f1038463bdaba2636a9491acb46a277550d9d392e9f98aab2a43d79e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"6442fe24306b709cd2915825463d36c18d25267ee1bd3c6a3fc8d3f372602919","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"6da31d750be1af3dbbe6e38b503e19dc84ce7f67730e4acb47ca85c9a3997672","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"6f5f7a1355e4ed73e51bb3f030f677880809624e0f8b786c393a351a2501219c","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"75c5bc63c99001fb0f2e8b2198e551808d3f6f57309385f288b8d942634a1c35","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"7df2c0e95ea58a049ed830fcca9c9880a7128409f67c869b53eea9387b2285da","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"82e73773d549ba957f2204af6d4d2c1cf170a7a161c00e04165cb68496e70837","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"86c60664512b45dfc964ed34fddadfd8c59b67915c486ec96cdd7da4f6c95f9c","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"8f9144d292126dc35d2c1f9809907292f12b4514db461a32b70a7d837967042e","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9a7da0741b30aa727c3e373095e1a0dd725bae4e61b8eb5638e69048b7203253","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9cd0816b849969b8416e43fcacbd8a3afe33dd8b72e802c42c355de1252b8ea6","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9fcb18d4baf46207c53d60b0635afad4482714533890bb48e436a254a5387cb7","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9fd8fda4123d9907523177f00d190d89e7f1ea8c687b4ca80c64e35cfefd2f5b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"a062845896766f82b5b6b43caec725177925d11140bb3f164d4f624bef40db4d","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"a5570161b8e3666ca7bfdfe88f809978e21dad7fe68a98b14aff3c246574fa43","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"aaed1413dee4877e1b256246048dd53e1f6b2dfba7ace865bb05935e5ca237b4","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ab86b2bd678ab3ef10ab68c466b7ecffe23bfb3a6ef5dda3abadfbfe4eaf9f55","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"b5b4a6f0378c43d1a2dc12c93438af0fd66b0a976463f2bebecb04c0d2a2965b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"bc974c6166c1205885fd7f678176787dbd1c4bec6c3b41b99aac04cf4e8b0438","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"cd677032a06805aa6da80e46d98a979f1d4459f91cd151d2bd701d7f53300c7d","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"cdf9bccd98bb3d1f4fbecd496356b9b4b2a6909b9cc5bbfcc35a4c6dec0be79f","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"d33db4510aad5ad39a18b96587e435da48fd02ce7f5d805a2a95ddbed99c55be","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"d844d68bf4a7e9dca03bc30788bc19d0b4bed38962bfc0a6b87e74a5926bfb9c","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"d971f81ea2c74ac5682a34fb0a1da00c06cc6e232238cf80a72d313131438d96","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"dfdb82b5e1768bb75ad1b5945dd0d4846e83397c37919c97b3cf70301d4beee0","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e05c6379c4a0811c95a6f224b6eb160bc7e8155848198ef8d53fc4e1e641d681","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e55ec4dc2559e274b43ba58584e2196b6488167e7b7f10b0081e0a9c39f29e17","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e6cca4e5ebe2824c588f245b4fcfc45c438b672a120705e75673912bad4df3ea","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"e97f5f31dc8603bf599703c5d2bb6392233b12e869441a56869a6ed9f3cfe6a7","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ea136f16cccb77cddc98cddc0a14a83b77bfecd278fefad03002866b67a2a785","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ecd414defcb93a1a62a5218474be07f4e6bdc602b84512c4e5b6a4c046f93ab1","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ef7e28288ab5d242a97c713ca8d4978be14e4746798d8fe8f1199a256df987ad","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"f7448a3ce1b674b582afc3ce637cc18aeda6a4b87548b38419f0fdbc6a961f0b","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"f8d57dbc64ef0cd306ea8787bddfb2291508c002358648a4f849e60b73624f16","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"f93f45f1ba356291f8e43d864d0c95ff1aae399d7e3555e18bb2fb08f56aef85","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"fa6930c5b5762130b890761a4a2f2b159ebdf252501d3b747e67c32a195f1212","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"fa98bbc082e6f0398dc6e7ed99f2366586b19c3e07e14afe3e1eac22a02a9e8a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"fb1584ba45514cfa19ff8297d90b87cce89225faeafea847902ff47540b55603","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"ff35b990ee10a0b7d2f378b4f77e12fee7a2c01cba6e29d35fccf29a3fc32400","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0234af5f580db6626f2a7da3e559fea14c85fbeae8d8bbaa9fe50fe1eadbce20","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"026cc77bd0118d0b0fb23c12ea4070b3adedb4901049309b0fbb5d343f23ef2d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0405121a11841816fbb5c37ec640295b707ead5e5e90afceec8c6d26d36b89f6","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"07446d7429225b418915a4cdd3209ca077c65a04e036a316910ed5111aae4483","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0790a6807f6fe23e4136d011f5d24b06e2e93d59f31b54f7ca8c074187cb14bc","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"0ffcd91712b8f7a0d5a6255d91431bac699403350a8dd152179a72a5407bd32e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1910d4ec68d7cdf87abad0e5482daa2725be756f3977ce0ec278b06eee95b9a2","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1ac6ce17bff60c7ab63cc5a0613de3bc95d0c885698dcb250d6e706d723721dc","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1edfbe3cd4abbbfa5bb64fbfda2b694dc27bdc2bf43485ed68fa4e009a41483b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20432c14ae8e2b382a936363590051ecced86fa2c7fc4fb3a142a6754ee3a8a1","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20c57c7f4f930cf7d2b679efc13ba7fbecf7f7907701b7050bc117fc055c41cd","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20dbbaa502a201960644c795ec041eda3e1e7f6b22ea6aecfb61d871d82f1f9a","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"22b87309be06b2c25c302beeb768905cf2e4feb40581075301f30e8175af4989","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"22dc20d764c053159f3d9bf81cb2e4491054266ca29cdfed267637828ffc5585","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"230b8c9bddef50a3a05a1f3de54a900d3b7d6faeb31633cda91c740dc4eb3df0","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"238297d169108c98a2d4626af8f21b19c4b6c7cf59dba7f3ee8ebc66431a7b58","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"354b1ed0687723f95732e53506d76b8935f43c4c5a3fc5d2ea8b6e032f27037a","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3795303d12ea1c3511d89431ccc5b137f0422812c4d6391c3dda69ba83bc5138","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3a03fcc14b33be029bf465926fbe4e3b50d9cbbd257c1329c1a09d6e3d51ce07","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3aa6f285ea3287c6e6b6efd85ec823aac3e8bb57b14ed667143e706c263cd02b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3eaaced785877507fe8e3ee6d7b20f5a9c0f3c1c9c3e1008644f14d126f02499","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"3fda7ea2b1167a41d35a76af005fe2e83fef0dd3e1f92adb38162bc326aa3e9f","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"4fdcf232f1620222aa8ebb4140f8ff9e4df4e5456bf486b7e9d595ad584bd299","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"55584a5d68238074127db736683181e52fad1bebe6e5c47227f063f1867bbcab","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"65da7551f02a1b04092f4424c3ac603b9678ab2e11d3a2eee871ff91691dc340","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"6ec94cb1c82c8e7d53280737a5ac81a87a2c3700c6756fe2e48135d340e23a34","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"7026d38b22f3f07287af1077384e1cabd2b9c060a726eced072baed3e746e2cc","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"738f282ac1b9e19589766dbd04203e8a353f699d5e20fd2a33231a5a43b2c55f","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"7443e6c90205f089a652d39f2c734b065ee2d31caae75c65c39849fa915121ca","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"794b65f9275a335fe93990764529043569ad0192f9cd5210127d164be6a02d17","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"7accfc45cea7730bbf56250febf86ba494e9aa48ac91ee6b6fdbe18c87125948","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"81f51f0b76d4d4a028609118e73e5d0dea40c40e11818d1448076a2a1459f305","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"82b0b1156dc53f9ba84cacc359e3ff611032fc16c9cb42d40cc22450542aca18","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"85aee33730d076723f6c34d4e6e85fcce12cd9a915dafdd0b0be11a764052b6e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"8940fa1c7065bc2d195ee4dde9f285ef8865512be349e563e8389f270a040c9d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"988a31a8c55bd029da7feb1d59601dae45a62370c0cc71affd91cce4f2c7ad95","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"9c96b70957f19ce12e95ab6707ca9a0ff1dd2b4398316345c774fd3ed009fe0e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a1ad804cbb749acb2715a93cddb0a4a8b005090394f8ba59823fd9abe662b7ad","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a38f6089ac07d5a08077f2a938fa9a9ce3187cf1d0945bf9427cd172c5b0edb7","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a79fbe141ddedfbbb015353053ca0abfdccd6e2f4aac398ef349ab02ea1d637c","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"a8b671a92f5e23c45cad8a1b8d273eb90f482db780e90e20fc073db8415882de","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ba10d5c256fa54a16211fadfae959953b61915e76bc9c818bdf27f141f48620d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"bb913418bcd18315762f2b7a90d5b2b627b3a803a94e0dbaf404fc37cf3c0e1e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c14a5bf416c111050f3c01f9eb312019d79dd48c33021d98a6a17be90bc94e15","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c176251ebeb4dd6ed7452613b62ca913760a464b5df07fbd38261c64a43a7f80","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c546460511702c57dce3c2a502e72728ac52f4e17ad01db4e2f1df9d85749433","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c9678a1d0afeb8bebc63313f1948bddf6056aa3f46f60242bdb863e10177e365","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"c9f831179ed94cf1dd3e33261efdae1890b83b92efba1fc3ab44d96ff0e83293","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ca8bddcb42c1ad62da6b28232b746387c70bc73f83c9bdaa522ac32aacbf5caf","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"cd9069a841fbe98bb39c002f92e470bd15b4f2fa478338c86af1fbadc560b0bf","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d0252d9787407ca82fc396a0e348b68f2ca460451e133e365d6b87728fae2ab5","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d2ef697ff7ce93a99a1ecb9bada994ad8774fef3f0ebcf089cd9c05331c620b3","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d31ea749491460d333cd226c9b483a00c486c27b079faa606afe6b5fa0405642","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"d92081da24f98bc6a986f0b560c3a578136d3424cc67dd149b67bcb0e1f29c21","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"dd98cd647e0fdcb1b2cdd255f2e7d94c26fa423165e9e3d0f2fec25bda4ee4ac","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"dee6843556c75cd55d8ca44339be5632c421daa76b7b87f6c9fbe6f53019f4c1","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e595a69b656f17d5869284ea95b799429d30cb1a24af721d8a7967774dccd0ab","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e6c1290ebdde46d8193a6106ebf8e29a72b8f152685a514a81e64984610d0f59","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e7b8feaf48ce77a449a907bda71e076340c6dd93920023c8fac9eac9fdba0998","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"e94cb2a361ab53f63cc6f93fd14b4a8b8bea7eae446cf32e755322e3fc12b81a","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"eab85837f796099b224575bfadd827e056270cf5a6a80ecba8184ab0e5932c88","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ed7e4f1ec742ff5b11ff9aaa80b8d076cb2a6a3976dfcadaffa293b39a5401f6","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ee6d73e71b3f4b0cd58bf4d5471102d6c9f5b892d3bc0a601029200dc711809b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"f01c77ea3e86ff9d0036332decf0cfe00a418350308feb18e16b12cedcbe2c46","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"f097960ef5ad5d2eda9874df327ff0fda3681cf5e790bf2effa00f0b0b3e8894","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"f5c4e42156185abc5aa27dd8331da5ae9c89eadaee3ef25f80a456bf5d31ee46","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"fd895cfafade17d9ab781eaf19976211ff70e49ab529d8e09ae289d6427dffe1","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":126,"runs_affected":17,"failed_runs":17,"run_impact_percent":23.943661971830984,"anchor_occurrences":126,"after_last_push_count":191,"also_in":["PROD","STG"],"trend_counts":[21,21,10,61,276,236,126],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":187,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":10},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with a private key vault using an available API version","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":10},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2884a5492ee0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dk2zvdznnrzn\" to finish creating: GET https://rp.j7618944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a5a009b-1b3a-4c53-8483-6562b10f6eb0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a5a009b-1b3a-4c53-8483-6562b10f6eb0\",\n      \"name\": \"9a5a009b-1b3a-4c53-8483-6562b10f6eb0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T22:28:55.765188822Z\",\n      \"endTime\": \"2026-10-07T22:57:56.06793791Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dk2zvdznnrzn--managed/providers/Microsoft.Compute/virtualMachines/r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dk2zvdznnrzn--managed/providers/Microsoft.Compute/virtualMachines/r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2884a5492ee0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dk2zvdznnrzn\" to finish creating: GET https://rp.j7618944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a5a009b-1b3a-4c53-8483-6562b10f6eb0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a5a009b-1b3a-4c53-8483-6562b10f6eb0\",\n      \"name\": \"9a5a009b-1b3a-4c53-8483-6562b10f6eb0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T22:28:55.765188822Z\",\n      \"endTime\": \"2026-10-07T22:57:56.06793791Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dk2zvdznnrzn--managed/providers/Microsoft.Compute/virtualMachines/r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dk2zvdznnrzn--managed/providers/Microsoft.Compute/virtualMachines/r1v2t1h5o2n7k4l-np-1-kbxtb-qjzfl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ckh4xm\" for cluster \"cluster-candidate-4-23-ckh4xm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2aa4ed1a5b40\u003e: \n    failed to create NodePool np-ckh4xm: failed waiting for nodepool=\"np-ckh4xm\" for cluster \"cluster-candidate-4-23-ckh4xm\" in resourcegroup=\"rg-candidate-4-23-g2p8crd8t6kf\" to finish creating: GET https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7981efbe-9393-4366-bd69-e17288ac49a7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7981efbe-9393-4366-bd69-e17288ac49a7\",\n      \"name\": \"7981efbe-9393-4366-bd69-e17288ac49a7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:35:08.39379338Z\",\n      \"endTime\": \"2026-10-07T21:04:16.844016594Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k to exist\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k to exist\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ckh4xm\" for cluster \"cluster-candidate-4-23-ckh4xm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2aa4ed1a5b40\u003e: \n    failed to create NodePool np-ckh4xm: failed waiting for nodepool=\"np-ckh4xm\" for cluster \"cluster-candidate-4-23-ckh4xm\" in resourcegroup=\"rg-candidate-4-23-g2p8crd8t6kf\" to finish creating: GET https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7981efbe-9393-4366-bd69-e17288ac49a7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7981efbe-9393-4366-bd69-e17288ac49a7\",\n      \"name\": \"7981efbe-9393-4366-bd69-e17288ac49a7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:35:08.39379338Z\",\n      \"endTime\": \"2026-10-07T21:04:16.844016594Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k to exist\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-7v82k to exist\\nMachine l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-g2p8crd8t6kf-candidate-ckh4xm--managed/providers/Microsoft.Compute/virtualMachines/l0u9z2p5c7y8d4r-np-ckh4xm-5hj7l-lx6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mcx5bq\" for cluster \"cluster-candidate-5-1-mcx5bq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x99848406080\u003e: \n    failed to create NodePool np-mcx5bq: failed waiting for nodepool=\"np-mcx5bq\" for cluster \"cluster-candidate-5-1-mcx5bq\" in resourcegroup=\"rg-candidate-5-1-tvjcv9r9dthd\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe8bd558-9f50-442b-bfc7-12311927703d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe8bd558-9f50-442b-bfc7-12311927703d\",\n      \"name\": \"fe8bd558-9f50-442b-bfc7-12311927703d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:23:04.012039216Z\",\n      \"endTime\": \"2026-10-07T18:52:11.882573758Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr to exist\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr to exist\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mcx5bq\" for cluster \"cluster-candidate-5-1-mcx5bq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x99848406080\u003e: \n    failed to create NodePool np-mcx5bq: failed waiting for nodepool=\"np-mcx5bq\" for cluster \"cluster-candidate-5-1-mcx5bq\" in resourcegroup=\"rg-candidate-5-1-tvjcv9r9dthd\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe8bd558-9f50-442b-bfc7-12311927703d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe8bd558-9f50-442b-bfc7-12311927703d\",\n      \"name\": \"fe8bd558-9f50-442b-bfc7-12311927703d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:23:04.012039216Z\",\n      \"endTime\": \"2026-10-07T18:52:11.882573758Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr to exist\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-ft2sr to exist\\nMachine q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvjcv9r9dthd-candidate-mcx5bq--managed/providers/Microsoft.Compute/virtualMachines/q2k6t7c1l5r1b0v-np-mcx5bq-wpgvp-pfw7r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107939651057618944","occurred_at":"2026-10-07T21:02:31Z","signature_id":"e81efffea3ea1b938a7a9d06760cc35a55e43a70882135723090d05ee5ffbc65","pr_number":6954,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"4afebe6cfa4becd64f3753581a33be535c50f0fc8ec471f7b9a6903b11fe2376","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"18f2e28a07e5eb92679cc517800744e5df3a152e2396fa940c051e7574885e9c","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"1f0e4f0c636adf2b731dd9281d3e7bb9c1c0e2048ac2b45a87fbf99ecbe04ac8","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"2926fb9d7060c87470cd426a61c704bac975307a608d031bbdacca06b4fc644f","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"4bd14a829b89ede2b97f05bf6929b7869177f362d89becc37f9d90e292a667b5","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"4dfde317b11106ca0bd23bf0cc1be0c00200032488cdff73f9587e90545288f7","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"76b920439a80aa38a9cd0875591b6edf3e0e9af781b1c8b022ae07a8fd86dbde","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"d076bde588818dceafe9295def496261167c5e627cf0dfa6e7e59d7fab7a0d26","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"df7306baffa1f0284a685605b0198dcfb8e6e6fa960112f72942b626ce31a529","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"e1f2efe03af68d4ce31e31e7c6ae84a90a95274374b7894b8be874df43aeb960","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"1c481c423bed0c16afe7ba8ce852af80db737e5d86b88d6594b69a5b2349734d","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"1eaca06a614138032740bad113fd6bc95fad2738fc2fec29290c9c84eb0162ce","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"4e7bbd17a69e19a522e72b5689c49118c5aad05be581a2713696272f925c70fa","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"52eaceaa1b46bb9ea1f0c0251416cb3da7851bb61c9a6e5565b5649beaf12298","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"69e74bb1a59bef22e2d3c9d8db69756b720248ab7d712391101dd02a7c60b033","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"75fa85c6eead75af9929e7654b5491890b965767e198093cbe942d232b71c61a","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"79dcae13bd7e35f5ef9c3691d9e87c155c8a8b1248ad3ddfff42d29fc452895b","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"93fb0fc62da8fba90112959c0e5524d7d28a492d00287a557d4996bd704526e5","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"98b46fb6c08d77f3f18d038b41ae82a882fc3d4628819ab2e9ea75b061cbd288","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"ac773143bc795433482c697f6cbd0e48f70363b2467a2da7e9de3148f8e2cd51","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"c967cd94b64d80ae0cc4a8a5f628e078837c44d0d953179ca8808c680873430c","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"cd3df0ae09c9ca011557e453c48b836e776c10198a4ff32ec4aa8ae20b7ae1a7","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"d7d9bc5f5ed38e25615efd7d9114e5c09fd421668e2a82dbf96d7fd13eaf6419","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"f4cde9518c1a88ddb8f822b81b4d7416f72ee540e165dfd3aae64bb95d75b1ec","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"110022798a0764c5024f8b28297ceca546ce14d0a1c6b612a61892cb5628f7a6","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"1b93fe9594667d36e6a2a6bd0616e1d6b11af83f9031d6f6ca44578ccd57caef","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"298862f500cc93b77a8c824b97595e4e37374b181407fc37157fd10b1e4c8c31","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"4c030cd147dc6473f9050e85fa4711eae9edb3e863c902974c185ecef7f0f633","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"4dfc950127ce86a016e64c08dd03052325b83c724046c516f0a0a51d07e6cde0","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"9199a30bc09dac1a956a7174474cc1782b878ab9069af756a8200e5c362e4a72","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"941be88d29bc7cff4b20af1db30253a5b8849dd7ba4250859ec3fffc8398f027","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"e5d7214ecd053506cec60d6039d445ec28a5ce3412453da44db155132462f979","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"fbf7fec41b640eac5fd46e05db1c159186d578dde36fe0f54035b78e8ce546f1","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"0fb6e1ec1717e82374bd0ab14045070f740a319efcfe84a5e1624839087834fb","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"59295f394d47135b361b2ed8985d6bc279f8687f208e575aab5859cb7bf31166","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"5d3064cdf6d821edd7ecbc32809672863b0adc7cf3ea6df16687482e8b85a923","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"74bc64dc70ad0017ff4f9b03b43959fd5f11246d6a15f16b814179375af6d735","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"99eb96d1396ba54566b3369529aeac2c545ab5eb91f2f84221e8279e27dafbf7","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"a76baf01b55b7a5ecd74c09bee77d30a3af38608bae3a4d5bbf78b1d074f8b8c","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"a90c96add63ec094880f02e255876531dc36c79c1f8d294ebdf378e9792415fc","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"c396caa9bcf0c2da1cd4662a8eb438222950ad162bc060be8a1addc8671b604b","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"c44c4d07f31f7026d27303c689b6f97d8b8cf333b83abef19daa07d09700c1c7","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"c97661e923fa1de8c63c33e700c6ecaf4ddc5c0588823453e627421ad9c0079a","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"0cde671c6e56522b67cc12ff4f25c7e4f0555299340490ca66e92acefbfbe1c1","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"15eae028ce53f84d13e63bb41284ec43f8011d73906a47c6d01bb1d26619a589","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"1602dd762444352f2f553c13ab6b7ee18d842ad9ec9a8b7c6c869612269efdf7","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"1bbd7ef56debff6d5c7488c79402871459f0f003e9957dbd25aa475bd5d012d6","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"2350e506ef8a86a7ebd95909719dcdd3df80ab16e1752475b084c284434ab98d","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"29781840512547ebdffe5b318cbf69d0eb9c139f7dd9a163309a42fd1518be5b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"3530c780c67a7e45eb884abbc05d52623a1cac06fd06d9407918744d15ce231c","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"6a98eed0f9f19d9b2606b420b4aa3c5f2ee0276ffd1335f89b2c5994956cf6d5","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"e4197d8f95eeed95cbec563c3d542bc9280009076d63cb82bbef9262337cecb3","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"ea20ed5ededb7561ed93b9f6df6f5b9b4077d95aa023417f6ee9312af308cb2b","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"05d8468ee56d53938c0f340e048e58d034a108f3f7a32001325ecf37dfe50d35","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"07a2a756e0c8a1bfef77b8bddb555d9e1545f1c96712630cd70c2290f204e24b","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"0c3e6b55ebab2f04a60faf61e054837580d8faa3d83e2cd44ac9868c9cfa8838","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"7eddaa92f651d8945d08fb4abbcb977dee0f718b2c4423dad1308f673ac0ffde","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"e4fa04f3828672393bb8ee717cf4607e3a078303082ff4844aa7b2729a55ffc7","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"ed20db877f6a2fe61c73e70aec12091b55b232eafae5e6f3a702af1d0ace5203","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"f5be6433f057fa85560c4273cfab71f2f16027337caee499a075f51ad97a2b5f","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"3a098b48b06940d26e0465ac20220cb5b7e73dd045646f72f9fbeeaccfdbf533","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"409c159a14de870f32d97bc50495e35950f682f737ef9607619128669adac534","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"5cf2a5571f1f56c265ec37f0381a1c8ed4008bb85ec06c4ae7cf57d61c36751f","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"60baf98859df1e164b9731da33e2dab18a29baa7c0dc9152363e95d9b3d8e605","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"843f6080d43311a72cff38f636f61f742f4dbbf3764932025820682bab88fbe4","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"9777b9adc2ed60d49bf0bb6bee7a613f4b848f9ce3762e434e392fb166d6f760","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"bce41bb751b8a42990d8eca8acf4cb9ef8037f2022db5e9570213dc8295a97a2","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"bdba72a5bf9a4e6a7d12a795d000a3b484b9987a912e0448d53f34ae618cfccc","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"2132eda40301f5eaf21d1b30205a9a5574c75a30280f0494851dcc8cf854d03e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"5359b5c88f9e8c98720978868a5088458008510399612cc6d979cfac3938551e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"750bbd1284409385baf6a12c22a9104af08af1e526abad4017570508d77bc277","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"936557c8acaeb1d7461dad4a5705bb52cb39c2b58ebc33b681671d3a2176e5ac","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"9aab43aa1fad33c45c2d9c115547e491f171c8e5dd8ebba2e023ceb28f5c022b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"b2e36904ecde082df7969eb49ba722cc1ebf740798096de32b1dabdda712327b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"c05743641b2b3d4c262aaccee2431126664657c73fe13d93acdb1b842b38ff45","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"dfc3f987c8a57f6e0531aa5fb1800c718ed85dc061fb6602bb86fe271cc56f17","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"ebffe6d6ecaf5e5873fbc8ae713b30c13868e7b43d7e1847d51ebfcdd987068b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107778619966230528","occurred_at":"2026-10-07T10:22:38Z","signature_id":"45abc7a27580a0d545fb5071d27064b3f18f5dbe32902c6602415496760c90ea","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"18b3fa7498379ee80ea560959f2e6233124f9b4a98d1ba7ce94dc24c4bd18bfe","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"28fd67d083a935f18b35824e91c321d3dc0a675a33cc2ad3042f20f7f21b2d18","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"29826f2dc72b792b387cb9569abeebe12cee5fb64b9656f64489111f62e4d0ec","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"39844fcc919184e3e26c50c19f16c1f74b28fd4bf3496ca74f197fb5b19b6431","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"39c083e3464a733a1377083d50da650490486daa38d016e0263bd9f0dd798841","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"42775d6d590be883e5cfd0afe00661abe77091dd685a801ceedb8574ff190602","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"84298de935fa96ecb1c7c68d067ce795f5ad49d83489fd349fcd2e4d24b108ae","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"8d589358c3bb0331b165597c6620eb42b56f46ffb021a70676a3457b789cfadd","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"a0edcc1432c7ec386284464c6dc9b384a6b36d43976ceb410c0dc0b24fec7f48","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"b06bc26557c2f7b7c1431f21a94f48d01c18df7244e082d80bf24ef9e3a7679b","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"c5054e34b231cffbef01421a9b31fafd7bd6e6472dd05a5f0e6b5ec197ea182e","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"05b82e80d394db855f72820095695dccb352d1ff5fc4eed9b28245e606c0d145","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"10c21f1ac4badc1c51061196b70c294168dac2e4ae5c1b9ec1c48bbe0e84ed74","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"3462fcf35a7e1ba371569d586044ac77401b9bd3ec7e4f2e4b22f1363c76be6d","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"50af757e09db672b178bd70b1adc1f876736fc58faca8db721bc68bbb1c417a5","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"696ec44a6df12b95105a2bc29ec5070faa7f6f41ba0f527c2fe0cd86b769e0fb","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"6f22a5b326045bc8517c0d65b469434082b3598a3c311cea397143588ae18d8e","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"c1cf9eb7afd6e93fb115c36a108273eeec62e2d5e0189af0f151c3e257eae80f","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"db54fa3dbd715eed9d612926ede9e164fd802289941471af853ad4a488bd084a","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"1add0a99393798752a0915fb4ea70862a58bcd3e033df28f52529f24f2f3a585","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"2170f480f911282c2abe5dc130e3b47c1b6faceca498b12db69544508b616cf0","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"2a51981b027d8aa97e4732f876ec757c350d36ec76746d72814156f84c2dfbdb","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"561ba19fb10ab687391aae0274302bec54070c7d29409470414b7dd1608f51f2","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"983fd3933dce71eb0669b8069c2aca9caa1b5d4aabd629fd7933c8491dc60b0d","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"a902f649acef68c24fbf4ded739d69eff3baf267586aaac8e393a75b25f0662f","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"df226f17583426298a4db5629842c56e1505c2567993c63de76bb3dd6c98edb0","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"e2a1db1bc07a51865a6d72643e3b66dfc46c1c837e30ee4392486f8f885f8d75","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"831b58931b42de25f29f96356e7160819ea0460206c7caf7b7aecd3a79744749","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"8cd5dff5cb797cdb21d04834eb3a07a380cb9ba6d257444312311bec337f797c","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"c8c45cb1becca3e1c26ed358851cc615189db8761433406668ac8b4a33f3428c","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"d81d269f8740f219f8eda6fcdc0709341c1d9b64d4c791777028838ce00216a8","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"df0ef413bf45345efe9139d1eb01f7179a89d40a680881f1fdcf15ebb2bbc0d6","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"f53952dfa7ee6849ad3e00f8bd97110d104593bd40f0b476fa6c5dc473697253","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"fd080a7568bfc33f70af0817e2c46e2e7ffa2e4df4898a7e33343134e0ec60da","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"fd5772fdac77f3ca1924151a6f31be6a24c58d4df40c7b6bbdb288bea1c86f68","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"319ff80a119cce276594c97e3e93d4509ad81c0e28a12bbc3e8fa2a597d14dc9","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"115fc9a87c835081d6747cb81d8e5f2ce1c931210978e133e13a437ef181c0f0","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"14d6394d0f9b51519a75c1fae3e72f083c452019ec69868fba057c34084a74e3","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"4a10651c6d339aec715190ab280a112cb4e3daa6943c976a5ab843f01fa2bf0e","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"6f1def9963421cbf1384fc961c05ecb86c555deca9a956b6cd51244bac971356","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"90299ee13eb97d04cd02aad44ba99f8edfe16f2814bc827bcc704513ec15ae9c","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"953b2c19892b92d0fc0c3df69d146d6f47f763be247d3757622c054f4b956a2d","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"95a1e8417d0a16f010d78b2112fce589c5e950c5099fee6d99af7f39eb186080","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"ac8e4fbbcabf95cc130fe5b4fc037bd85d7b0106da3b7a49d23bbaba2d73569a","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"b8a0763536367e04cd9eb8cf0ed07c7979f2cde3281236d8e855a4932da00c7c","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"cfb3c0aeffb83e972da151272b81e096b174adbbdd1f6621c8ee47c5a32d001f","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"ee750420154f79a00dca045668ab90bb645a634d33016eee21f6ea06864c7b62","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":29,"runs_affected":29,"failed_runs":29,"run_impact_percent":40.84507042253521,"anchor_occurrences":29,"after_last_push_count":58,"trend_counts":[5,19,15,3,26,15,29],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":172,"prior_last_seen_at":"2026-10-04T23:41:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":29}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-08T00:05:01Z\n  Ended:       2026-10-08T00:11:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6670976-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T23:44:13Z\n  Ended:       2026-10-07T23:58:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6213760-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T21:16:57Z\n  Ended:       2026-10-07T21:22:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3227776-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7417/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107967978476670976","occurred_at":"2026-10-07T22:55:04Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7417,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107964840596213760","occurred_at":"2026-10-07T22:42:36Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107926490833227776","occurred_at":"2026-10-07T20:10:13Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107926353130033152","occurred_at":"2026-10-07T20:09:40Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7305,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916465603612672","occurred_at":"2026-10-07T19:30:23Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107914072107257856","occurred_at":"2026-10-07T19:20:52Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"bb0b7baeaee3903b2c32b416b3b8a2cd08cc9041556204906cc5c3c0dbf4e3a4","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"bb0b7baeaee3903b2c32b416b3b8a2cd08cc9041556204906cc5c3c0dbf4e3a4","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7322/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875761300967424","occurred_at":"2026-10-07T16:48:38Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7322,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107872947002675200","occurred_at":"2026-10-07T16:37:27Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7410/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107844183950626816","occurred_at":"2026-10-07T14:42:46Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7410,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"f0fdfb1a7d35edcb5a2bf5e1776a6f8480ed63182bae2c515bbe819b35402c27","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"bb0b7baeaee3903b2c32b416b3b8a2cd08cc9041556204906cc5c3c0dbf4e3a4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107802095104036864","occurred_at":"2026-10-07T11:55:55Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107778619966230528","occurred_at":"2026-10-07T10:22:38Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107752436901875712","occurred_at":"2026-10-07T08:38:35Z","signature_id":"7a43480e65ff97585b06bb990026d9f475bc404b7071e50eb5d7c926e16db6df","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751899905134592","occurred_at":"2026-10-07T08:36:27Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7403,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750854802018304","occurred_at":"2026-10-07T08:32:18Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750168211230720","occurred_at":"2026-10-07T08:29:34Z","signature_id":"33026751834bf5223e46b5e0fa41f4843e6b143bf0d32bcaa237cc03345d8d5c","pr_number":7370,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750153682161664","occurred_at":"2026-10-07T08:29:31Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107680144293367808","occurred_at":"2026-10-07T03:51:19Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7097,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":23,"runs_affected":10,"failed_runs":10,"run_impact_percent":14.084507042253522,"anchor_occurrences":23,"after_last_push_count":49,"trend_counts":[0,0,0,20,78,57,23],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":79,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c574774040\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-m842v74xwlhh\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"name\": \"310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:10:29.82492819Z\",\n      \"endTime\": \"2026-10-07T18:39:31.852478192Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-2tvj4-977r6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-977r6 to exist\\nMachine v20261001-np-1-2tvj4-xn4zw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-xn4zw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-2tvj4-977r6: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-m842v74xwlhh--managed/v20261001-np-1-2tvj4-977r6-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-2tvj4-977r6-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c574774040\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-m842v74xwlhh\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"name\": \"310e7732-14f1-4d4c-820c-fb521ce69b9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:10:29.82492819Z\",\n      \"endTime\": \"2026-10-07T18:39:31.852478192Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-2tvj4-977r6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-977r6 to exist\\nMachine v20261001-np-1-2tvj4-xn4zw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-2tvj4-xn4zw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-2tvj4-977r6: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-m842v74xwlhh--managed/v20261001-np-1-2tvj4-977r6-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-m842v74xwlhh--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-2tvj4-977r6-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d8b7da68c20\u003e: \n    failed to create NodePool np-74nbzb: failed waiting for nodepool=\"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\" in resourcegroup=\"rg-candidate-4-23-z987ndt4v67c\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"name\": \"ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:12:58.862762914Z\",\n      \"endTime\": \"2026-10-07T18:41:59.529725798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97 to exist\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Network/networkInterfaces/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '2' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2d8b7da68c20\u003e: \n    failed to create NodePool np-74nbzb: failed waiting for nodepool=\"np-74nbzb\" for cluster \"cluster-candidate-4-23-74nbzb\" in resourcegroup=\"rg-candidate-4-23-z987ndt4v67c\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"name\": \"ba9ec6af-5a25-490d-a829-97ac1ecf6a0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:12:58.862762914Z\",\n      \"endTime\": \"2026-10-07T18:41:59.529725798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97 to exist\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Compute/virtualMachines/y2i2u2o9d4w2c0t-np-74nbzb-r688d-k8q7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z987ndt4v67c-candidate-74nbzb--managed/providers/Microsoft.Network/networkInterfaces/y2i2u2o9d4w2c0t-np-74nbzb-r688d-22d97-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '2' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2af3a314160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-gvbd6v968qsn\" to finish creating: GET https://rp.j8893056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"name\": \"0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:57:49.24099524Z\",\n      \"endTime\": \"2026-10-07T18:26:50.588266562Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-vh9lt-cmt4b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-cmt4b to exist\\nMachine v20261001-np-1-vh9lt-j5d2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-j5d2x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-vh9lt-cmt4b: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-gvbd6v968qsn--managed/v20261001-np-1-vh9lt-cmt4b-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-vh9lt-cmt4b-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2af3a314160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-gvbd6v968qsn\" to finish creating: GET https://rp.j8893056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"name\": \"0f0eaa3c-f753-4064-9736-bc1f28a4ed88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:57:49.24099524Z\",\n      \"endTime\": \"2026-10-07T18:26:50.588266562Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-vh9lt-cmt4b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-cmt4b to exist\\nMachine v20261001-np-1-vh9lt-j5d2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-vh9lt-j5d2x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-vh9lt-cmt4b: Failed: interfaces failed to create or update. err: failed to get existing resource v20261001preview-gvbd6v968qsn--managed/v20261001-np-1-vh9lt-cmt4b-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-gvbd6v968qsn--managed/providers/Microsoft.Network/networkInterfaces/v20261001-np-1-vh9lt-cmt4b-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to https://aka.ms... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"f3c9ccebcdb83b8eb8b3305c00393328a72b529c5a2452faaa135e95a8164391","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"3111f51ae03691646571b56f93f83f1a4aa75f6307d6e356732ae922fa08cb09","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"3823b66b09bfd6e9e4ad92f72a9d4f3ee3c224b37b1c8f1f8ebda27b147d7c46","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"7649741d4e6001a4a976b6c337ded5372079a83044d0f0df24c3502c5d934dc4","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"94460068fa46dffaa7a997415d9e364ab28771906e158a3e989a36795e455e7d","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"f5aa64a25fb502f45efe4709b5d67fc37312041962fb3ca94ee6c6d91283814a","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"87b8ed371a8aaa18808fd97c59919f61f332deb12ffa10d0bd4e46b72315a96e","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"0e2781c957524dd582ad9d58b73f8540584537b1abf21b9ba0da2dd49354d02d","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"520474359be0f062cf8b98378dd4cc9e921db0bf50d302b2dd3eb02b1d62e274","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"e523689aea35fd7e6b880b1921b71e1d38fcc318168540029872f15f62fe2771","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"2522196ffab484807654ebdc0cf7d0567bc80e6f3f38b2482ddb3bceb953bbe5","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"33f522deb518c9faff818f3ebc3464e66c1d30a6e753a3a196af8f13e31f4624","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"7d1cd6af866cf990e6b26ab81fc3a9346742d73a2e9757eb55f2c7565d966db4","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"128685f67ffac53964edbe30eb85e252710a82d7325bb00e9be19d64a0dfb550","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"bdc85c73bc1b93540460da38830900bf54f131c55a8128f1abd00806c4591690","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"0e5eed768e639c704016ff6d0901d55fa558c49fd15dda10ad7f2477cc4d65c3","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"44d25dd92f2c1f9cc76ea6b88a4f23c16c0ebb713bd0c5d55d9a1d08edf04059","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"99b003f7a7a441d509d8bef27060bc681040db112029ece09584795fb10dadfd","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"41f661f65c04dcc379b400b33f94f5db1cca5ca24f1ddb4db781f3990be4d132","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"734ef25191ec5fa5c8dcbe8786f5f56d52fcdf12b3ec9ca735b1f1776ac9e851","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"0de4e74aa7302328587b11276c0a2114131a5a3b2f9d1227450eae431fac0c74","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"1b3a05a3a5dbb0b03df2aaec8331ce4285abcace77601310a4833563c35a01b4","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"92f959746d1f89cf28cbdaf39ae77041de5bfce5280761a1e7841543df6376fc","pr_number":7371,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":16,"runs_affected":12,"failed_runs":12,"run_impact_percent":16.901408450704224,"anchor_occurrences":16,"after_last_push_count":21,"also_in":["PROD","STG"],"trend_counts":[0,4,0,5,36,39,16],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":85,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2de8dabda108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-v5jj9lf2wr67\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"name\": \"c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:20:36.247617787Z\",\n      \"endTime\": \"2026-10-07T18:49:41.673001167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2de8dabda108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-v5jj9lf2wr67\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"name\": \"c17139f2-d108-4953-bf0d-a162f95b9ed1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:20:36.247617787Z\",\n      \"endTime\": \"2026-10-07T18:49:41.673001167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zwd2g-2w92c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-2w92c to exist\\nMachine cilium-cluster-cilium-np-zwd2g-xq8t6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-v5jj9lf2wr67--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-zwd2g-xq8t6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x259bbf69a258\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6m6qldl4t7ln\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/74400ec9-b7cc-4ceb-933e-5f049cfbef5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/74400ec9-b7cc-4ceb-933e-5f049cfbef5c\",\n      \"name\": \"74400ec9-b7cc-4ceb-933e-5f049cfbef5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:16:07.02818237Z\",\n      \"endTime\": \"2026-10-07T18:45:11.860346136Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-72bgn-5qwj8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-5qwj8 to exist\\nMachine agg-cluster-agg-np-a-72bgn-msjpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-msjpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-72bgn-5qwj8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-5qwj8 to exist\\nMachine agg-cluster-agg-np-a-72bgn-msjpl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-msjpl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x259bbf69a258\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6m6qldl4t7ln\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/74400ec9-b7cc-4ceb-933e-5f049cfbef5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/74400ec9-b7cc-4ceb-933e-5f049cfbef5c\",\n      \"name\": \"74400ec9-b7cc-4ceb-933e-5f049cfbef5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:16:07.02818237Z\",\n      \"endTime\": \"2026-10-07T18:45:11.860346136Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-72bgn-5qwj8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-5qwj8 to exist\\nMachine agg-cluster-agg-np-a-72bgn-msjpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-msjpl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-72bgn-5qwj8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-5qwj8 to exist\\nMachine agg-cluster-agg-np-a-72bgn-msjpl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6m6qldl4t7ln--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-72bgn-msjpl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x14bbdf4ea0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-db8t5zc8d27b\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/28c778fc-2439-43ac-8817-96efd7f98d9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/28c778fc-2439-43ac-8817-96efd7f98d9b\",\n      \"name\": \"28c778fc-2439-43ac-8817-96efd7f98d9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:20:23.837365353Z\",\n      \"endTime\": \"2026-10-07T18:49:29.442401902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-sxrv8-p75dc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-db8t5zc8d27b--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-sxrv8-p75dc to exist\\nMachine agg-cluster-agg-np-a-sxrv8-whnvh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-sxrv8-whnvh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-sxrv8-p75dc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-db8t5zc8d27b--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-sxrv8-p75dc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x14bbdf4ea0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-db8t5zc8d27b\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/28c778fc-2439-43ac-8817-96efd7f98d9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/28c778fc-2439-43ac-8817-96efd7f98d9b\",\n      \"name\": \"28c778fc-2439-43ac-8817-96efd7f98d9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:20:23.837365353Z\",\n      \"endTime\": \"2026-10-07T18:49:29.442401902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-sxrv8-p75dc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-db8t5zc8d27b--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-sxrv8-p75dc to exist\\nMachine agg-cluster-agg-np-a-sxrv8-whnvh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-sxrv8-whnvh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-sxrv8-p75dc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-db8t5zc8d27b--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-sxrv8-p75dc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"6606f0c53685d57eac73a261b99ec8523294d02e9f04d706f5486f1b88272365","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"e6140a91cf58e1093a15df9527c4955caccd5e340b860748578f39c12af14e5e","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"5ceb481507f4ca52c699d4ee4b5bc627642caa14a2e4b131a7ee8c113d7edf2c","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"16e437d19740573dbce777293dfb290c1261ca41fa7793782e50c0650b0ff035","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"83affcfb11dd475414f0e926b53f754c59b86fee5dfc1b34515c53c67abd39b8","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"5c68c137d6e10f757f06aa245dbad8a30058f94e21fab5889b2c912a02342357","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"57b1884bac955ab5f382942dfb3a03cdad30c1cd1fa31f46943b54e6a0ebe071","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"c0d8513a01d197d2a0670900409b40efbe2c6c80538e5b9afafd3a6a0d810067","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"9d1a449c54137967e0ef922ae6fb2f6b6819636ed301e6416ddd77db7dab6415","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"42242f0736ff6b2cabbb937ba4f48e4f3ed306ea01fcfdba4459ac3cfb832b7f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"7b412a98d9ede076c1a2df7a7038043eb6df6fa1023a0a7a3882d3027c3aaf83","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"ea119913577e1401aead2826975e251b0b4dfe27af9566d73b22dca408a22619","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"4b1fb5b920b17720b82999c711081f2a10ec1c9d9ea8ed05033b5998a486bb39","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"ee6bea4d8f68ba09c2d18707029359aa47b377c9594af345abe1ae957945edfd","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"83b0bcfc35f4e8dd7f58f5584c061cc026360328425f7ff053e8d20e993e57f4","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"e71983d5997d779d858dc33d7be7f632aa394ef38c01af01f25da4318397c4ab","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-khpl5jtvh4pb\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":15,"runs_affected":14,"failed_runs":14,"run_impact_percent":19.718309859154928,"anchor_occurrences":15,"after_last_push_count":3,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,0,15],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":106,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x37d635eafd40\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lpkn85dzg6wg\" to finish creating: GET https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/321127fe-bb12-4e32-b036-dd251357b191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/321127fe-bb12-4e32-b036-dd251357b191\",\n      \"name\": \"321127fe-bb12-4e32-b036-dd251357b191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:56:46.02075406Z\",\n      \"endTime\": \"2026-10-07T21:25:46.806941683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lpkn85dzg6wg--managed/providers/Microsoft.Compute/virtualMachines/m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lpkn85dzg6wg--managed/providers/Microsoft.Compute/virtualMachines/m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x37d635eafd40\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lpkn85dzg6wg\" to finish creating: GET https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/321127fe-bb12-4e32-b036-dd251357b191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/321127fe-bb12-4e32-b036-dd251357b191\",\n      \"name\": \"321127fe-bb12-4e32-b036-dd251357b191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:56:46.02075406Z\",\n      \"endTime\": \"2026-10-07T21:25:46.806941683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lpkn85dzg6wg--managed/providers/Microsoft.Compute/virtualMachines/m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lpkn85dzg6wg--managed/providers/Microsoft.Compute/virtualMachines/m3p5o0q6k9u0x5e-np-delete-main-kckl6-dqrv2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2a2d2cf153c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qtdjcjq4g46d\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d8ae7058-dd29-49ab-b721-9f8daeb7b0c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d8ae7058-dd29-49ab-b721-9f8daeb7b0c8\",\n      \"name\": \"d8ae7058-dd29-49ab-b721-9f8daeb7b0c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:35:04.887969322Z\",\n      \"endTime\": \"2026-10-07T19:04:11.73701564Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g to exist\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g to exist\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2a2d2cf153c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qtdjcjq4g46d\" to finish creating: GET https://rp.j3880064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d8ae7058-dd29-49ab-b721-9f8daeb7b0c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d8ae7058-dd29-49ab-b721-9f8daeb7b0c8\",\n      \"name\": \"d8ae7058-dd29-49ab-b721-9f8daeb7b0c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:35:04.887969322Z\",\n      \"endTime\": \"2026-10-07T19:04:11.73701564Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g to exist\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-6xk2g to exist\\nMachine c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qtdjcjq4g46d--managed/providers/Microsoft.Compute/virtualMachines/c1m0g4d8l1a5k7p-np-deletable-d2gzm-rjncx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xf6ccd9b14e0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-b4g4j9tj9qn5\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f1cecb5-6672-47d6-91bf-bf444c657b24\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f1cecb5-6672-47d6-91bf-bf444c657b24\",\n      \"name\": \"6f1cecb5-6672-47d6-91bf-bf444c657b24\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:34:53.377476522Z\",\n      \"endTime\": \"2026-10-07T19:03:57.309891257Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8 to exist\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8 to exist\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xf6ccd9b14e0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-b4g4j9tj9qn5\" to finish creating: GET https://rp.j6440704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f1cecb5-6672-47d6-91bf-bf444c657b24\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6f1cecb5-6672-47d6-91bf-bf444c657b24\",\n      \"name\": \"6f1cecb5-6672-47d6-91bf-bf444c657b24\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:34:53.377476522Z\",\n      \"endTime\": \"2026-10-07T19:03:57.309891257Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8 to exist\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-l4bt8 to exist\\nMachine w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-b4g4j9tj9qn5--managed/providers/Microsoft.Compute/virtualMachines/w8m9b4n1w5y7h6s-np-deletable-686hg-zqww5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"324a1ddefe06745af54ceec3a7cd753b13b17acc2674cc6993eade1b19c7defe","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"7701741dd3b01d9acfabc93cc96d6cf9208adff0af9d2c49651fc30cf2762efd","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"4f28c03f3ffd6f1feafc08f8ba7f2591ca3edbaf7ccebb94d84d8a4050b8d5fe","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"99d703714a8db74b79f0a7f4a791a2c67f0504f0ca749d98ad3677bf89e16f69","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"2f81340bb3a651933dc821aa2ca86669271cdc50691e70e09c6e8aa62b4e76ff","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"8da4047bc4d33387f4b311f92cb1e8ba3a4d59fa7021a9ff0423a80376528f27","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107873993380859904","occurred_at":"2026-10-07T16:41:37Z","signature_id":"07ef48ddd58323e88a3c6d921a967ff2a51796e4b4cdf8b03105202a4a8fcc97","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"00d0b2d32441e274eba0d75f1dfcc7751b4d1e6cdc39d82b613431282652086c","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7404/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107834229550223360","occurred_at":"2026-10-07T14:03:36Z","signature_id":"59ad17797e4c5c9a4a361e9396ecaa021b1de278966eb1d51526ddfdeae7e5fc","pr_number":7404,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"9c9a5d0f6ef88f93fb5706cfbc62086c704621ae3953320715626c1893a4dfb8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7405/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107765914286952448","occurred_at":"2026-10-07T09:31:59Z","signature_id":"54b9ecb4dfa7c1ec53a4c0d5518fdcc384dc08d32eb42945364fb76f1fdcc511","pr_number":7405,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"37771380d1cfd06af04bc60a3e3b334305eddaf4e748f94656e95d0becb4a8ed","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750792218808320","occurred_at":"2026-10-07T08:32:03Z","signature_id":"9e15c77d960faeec1308565e014df7f8752b6fadfa234829868e7060696eefa5","pr_number":7399,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750686056779776","occurred_at":"2026-10-07T08:31:38Z","signature_id":"a0d80e080c16f8391f653b4f34a80f5db542694bcda517ded36abdc1bf6f5abf","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"a2f41ec8c555bd7bb9e44081752bfbb6e97ad5970facf3a5de491297baa93884","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":12.67605633802817,"anchor_occurrences":9,"after_last_push_count":68,"trend_counts":[6,5,2,0,3,6,9],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":152,"prior_last_seen_at":"2026-10-03T16:57:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":9}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T21:50:06Z\n  Ended:       2026-10-07T21:56:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3227776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ta6nelbkbeh1s64mru6t7ohgsjall8a-h3w3a5j6l2a4p9s\", pod=\"openshift-apiserver-d5448bdcd-ltz5s\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ta6nelbkbeh1s64mru6t7ohgsjall8a-h3w3a5j6l2a4p9s/openshift-apiserver-d5448bdcd-ltz5s has been in a non-ready state for longer than 5 minutes.","alert fired 9 time(s) (2 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T20:35:02Z\n  Ended:       2026-10-07T20:41:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7631744-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ta5emb000bsfom310889hpe5casu1jr-v5k9k8q2w5x0q6v\", pod=\"openshift-apiserver-b9fcdfdb5-55mlh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ta5emb000bsfom310889hpe5casu1jr-v5k9k8q2w5x0q6v/openshift-apiserver-b9fcdfdb5-55mlh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-07T20:35:02Z\n  Ended:       2026-10-07T20:41:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7631744-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ta5fdlm2qo2eb32ju5uasrq631306b1-cilium-cluster\", pod=\"multus-admission-controller-578b7846f7-h57x6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ta5fdlm2qo2eb32ju5uasrq631306b1-cilium-cluster/multus-admission-controller-578b7846f7-h57x6 has been in a non-ready state for longer than 5 minutes.","alert fired 6 time(s) (3 unknown, 3 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T18:11:35Z\n  Ended:       2026-10-07T18:21:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3880064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ta3l68gqrcq7gs6igibvm68inmua3g6-u0t5l5t9k7x3r9l\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ta3l68gqrcq7gs6igibvm68inmua3g6-u0t5l5t9k7x3r9l/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-07T18:11:35Z\n  Ended:       2026-10-07T18:22:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3880064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ta3kutmkbpo2kifvv50cfrikfbop2vu-y5c5w0x7t8g5d0d\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ta3kutmkbpo2kifvv50cfrikfbop2vu-y5c5w0x7t8g5d0d/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-07T18:12:33Z\n  Ended:       2026-10-07T18:22:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3880064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ta3l4ibb8m6kaqmai8gv3qag7e9qrug-t0m8d4r3h1i2g1k\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ta3l4ibb8m6kaqmai8gv3qag7e9qrug-t0m8d4r3h1i2g1k/etcd-1 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107926490833227776","occurred_at":"2026-10-07T20:10:13Z","signature_id":"74818e91a0caa04c1765b38bbad74a23ca21673b3ee1229ea2981f98f72f1e52","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107908786537631744","occurred_at":"2026-10-07T18:59:52Z","signature_id":"156501ac5602097c0938c316b8717eec9df92b585d0d3f3819bbc868a6086146","pr_number":7399,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107883389313880064","occurred_at":"2026-10-07T17:18:57Z","signature_id":"1c9a57ae155d22074516444f9bdd42c3eb77f6c4185d2704e7d5ec64885ce837","pr_number":7248,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7411/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778164936704","occurred_at":"2026-10-07T17:04:36Z","signature_id":"e58cbbeedba7967d85e44f7a9a6e4300651d973f923a375d1899466c0cf4ac3e","pr_number":7411,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7410/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107844183950626816","occurred_at":"2026-10-07T14:42:46Z","signature_id":"260ad305c6a9edb3a0d9ff7ae59c6ed719a99dd03de1fa7d3c3022010b26b23a","pr_number":7410,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"5099bcc506260d26581bf59530494d1a9f6bc90b5796a9d216da2138646a67c4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"ece3af58d3fb6cba54ec581a6635f6e1e2add57872da32b2115f2e616adaf1af","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107717187903950848","occurred_at":"2026-10-07T06:18:31Z","signature_id":"753a35995836a60d7340477581bda7ccd89cec93b1ccc0f139677a210c813f54","pr_number":7402,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"5211917d4010780a3db4438b5c6318344f5122605ba573b6a6b2cc8edda6c262","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-brq8pbwd25s6/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":7.042253521126761,"anchor_occurrences":5,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":16,"prior_last_seen_at":"2026-10-04T22:58:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-zmb9zlf6bqmq/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-zmb9zlf6bqmq/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-w8w2hprnrv4g/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-w8w2hprnrv4g/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-cbtpkz6skzqd/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-cbtpkz6skzqd/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7415/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107928469479362560","occurred_at":"2026-10-07T20:18:05Z","signature_id":"03a6301221c7a6825f5fed75697ef820c87281a3d0a2187bc94a8af35a2207fc","pr_number":7415,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"beeeb8557a906e3bf74511fcdf63017c044aed615160f6a1189ef1547010f8e9","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107836958540566528","occurred_at":"2026-10-07T14:14:27Z","signature_id":"8f4a370df9d0045874c7c3eada9b5a2a1b6adf39b052fcfcd9941efab5980562","pr_number":7227,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107769248779603968","occurred_at":"2026-10-07T09:45:06Z","signature_id":"cdc7d229df7856f7d9198b7ba1b064f7154c9b9b742a95ef9a1ec56ca43c04cb","pr_number":7398,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"cad96447c550c7c633d7082b5c5d1b62094008a4833722c26bba79211fa96628","pr_number":7310,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.633802816901408,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-29T20:58:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1e8a40f9d110\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1e8a41056380\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x1e8a40efe990\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1e8a40f9d110\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x1e8a41056380\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x1e8a40efe990\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x21f6a70e8e10\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21f6a69d2ce0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x21f6a85f6620\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x21f6a70e8e10\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21f6a69d2ce0\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x21f6a85f6620\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2b3cb6331890\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2b3cb6202180\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x2b3cb57e7010\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2b3cb6331890\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2b3cb6202180\u003e{\n                msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n                err: \u003c*errors.errorString | 0x2b3cb57e7010\u003e{\n                    s: \"IngressController loadBalancer config is nil\",\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107970662109810688","occurred_at":"2026-10-07T23:05:44Z","signature_id":"94db297794b5684fabf014b75d93616ed7e1a167578aa35ce2524a04ff1dbadd","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"787ad618025d031d81ee980f6b67a1bd19c9afd249e15e2fd07410bfe1d4ae20","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"50c5be6aacbbb8af14d88b4bc3038a15509228c93ec4dc6e779b9e4c9fd79f5a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1faf7d5cc80b0f7cbde2eb8dbba3d93237d698cd5407434495000a5ca371a54d","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.633802816901408,"anchor_occurrences":4,"after_last_push_count":8,"trend_counts":[3,2,0,0,6,5,4],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":26,"prior_last_seen_at":"2026-10-02T20:20:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":4}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T02:13:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j5347456-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.24:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347456/providers/microsoft.monitor/accounts/services-j5347456\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-8fb68546b-b5vcf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j5347456-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T02:14:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j5347456-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.24:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347456/providers/microsoft.monitor/accounts/services-j5347456\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-8fb68546b-b5vcf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j5347456-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T19:18:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j6440704-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.112:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6440704/providers/microsoft.monitor/accounts/services-j6440704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6f8d84d4d4-wkcql\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j6440704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T19:18:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j6440704-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.112:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6440704/providers/microsoft.monitor/accounts/services-j6440704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6f8d84d4d4-wkcql\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j6440704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:35:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j9974528-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.34:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9974528/providers/microsoft.monitor/accounts/services-j9974528\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6c57d99f8b-ljtj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j9974528-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:35:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j9974528-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.34:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9974528/providers/microsoft.monitor/accounts/services-j9974528\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6c57d99f8b-ljtj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j9974528-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107979694195347456","occurred_at":"2026-10-07T23:41:38Z","signature_id":"a9f42fb8cd948b493ad952c111be2d077f8870c40915de4addfd8f530e496344","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107880778506440704","occurred_at":"2026-10-07T17:08:34Z","signature_id":"7263867785ff09df80dda0390383f3d5a769b78cdf799960e2fb0de6363a6a4c","pr_number":6807,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"63861b0edd2874dd78327b3b31697a4dfd50630052ae8d5d23132d2a7630b468","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"1a734d7fe3d2c29c8e29a73b6bae5924a4a747ab2baa59c0a15b4bb9931650a5","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.225352112676056,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-03T00:54:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-ff6jdg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d44c09e520\u003e: \n    DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x17d44c09e500\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-ff6jdg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d44c09e520\u003e: \n    DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x17d44c09e500\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7dms2.apps.aro.i7t6k9j8s5g2b2z.byrq.j6670976.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-vdz5wq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x68ffbb0c4e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x68ffbb0c4c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-vdz5wq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x68ffbb0c4e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x68ffbb0c4c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5fr7p.apps.aro.v2a6f1z0z3t6f6o.v5i6.j0890368.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a03aec70780\u003e: \n    DNS for route host agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2a03aec70740\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a03aec70780\u003e: \n    DNS for route host agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2a03aec70740\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g9blz.apps.aro.u2q5q2w5k8k4z5y.u6hb.j3950848.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7417/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107967978476670976","occurred_at":"2026-10-07T22:55:04Z","signature_id":"117275c46ce91f05ea60621ea4c13a1b949e2cbb5298e2ee95b6e289338fe062","pr_number":7417,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107739528360890368","occurred_at":"2026-10-07T07:47:18Z","signature_id":"d617fd03098208e530c7ebbec73454d1787eb1156909d4b1d261a619471781dc","pr_number":7310,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7402/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107717187903950848","occurred_at":"2026-10-07T06:18:31Z","signature_id":"c0b3902507c3f57210df372dcd211846b078fe3f061059eb55d1e7f21f3822f6","pr_number":7402,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.225352112676056,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,5,6,3],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T01:40:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1217408-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.110:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1217408/providers/microsoft.monitor/accounts/services-j1217408\", namespace=\"klusterlet-2ta9fcfdh71dkv4knt12m1ug0ranvkol\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-58rr2\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ta9fcfdh71dkv4knt12m1ug0ranvkol on cluster ci01-j1217408-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:39:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j6520192-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6520192/providers/microsoft.monitor/accounts/services-j6520192\", namespace=\"klusterlet-2ta7n5c8id8chjml49ukhi22658fcgoj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-lkfg4\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ta7n5c8id8chjml49ukhi22658fcgoj on cluster ci01-j6520192-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T21:31:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3612672-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3612672/providers/microsoft.monitor/accounts/services-j3612672\", namespace=\"klusterlet-2ta5r5t5b117do0a5grfaj0nbn80qof7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zcpkv\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ta5r5t5b117do0a5grfaj0nbn80qof7 on cluster ci01-j3612672-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T21:42:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j3612672-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3612672/providers/microsoft.monitor/accounts/services-j3612672\", namespace=\"klusterlet-2ta5r55d86kstjo2bp5n4d3bvb279pda\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zcpkv\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ta5r55d86kstjo2bp5n4d3bvb279pda on cluster ci01-j3612672-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107979674851217408","occurred_at":"2026-10-07T23:41:33Z","signature_id":"55900fd3bc28e32d9a639ce8648efcc35e3932eddeb80af6e6b043605ce1b905","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107955248256520192","occurred_at":"2026-10-07T22:04:29Z","signature_id":"b6691d6bcc2d313c3a62c74fbf150224b295d130f055782982141b1f34547850","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916465603612672","occurred_at":"2026-10-07T19:30:23Z","signature_id":"8869545ed690b35be454db0cd6dff2e92fefd57b72a3ff6445d159a8e93031e9","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-28T08:18:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0xf53ccb2c090\u003e: \n    resource group \"rg-cordon-test-zfbgp6td9bvg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-zfbgp6td9bvg\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3df7278a-fc9c-4dad-bb05-8577c80b31f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3df7278a-fc9c-4dad-bb05-8577c80b31f1\",\n      \"name\": \"3df7278a-fc9c-4dad-bb05-8577c80b31f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:55:02.189604085Z\",\n      \"endTime\": \"2026-10-07T18:19:08.046890686Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3aagkkq4l3ogcvf4h8o101v6q3m76 still exists (deletion dispatched at 2026-10-07T17:55:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0xf53ccb2c090\u003e: \n    resource group \"rg-cordon-test-zfbgp6td9bvg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-zfbgp6td9bvg\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3df7278a-fc9c-4dad-bb05-8577c80b31f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3df7278a-fc9c-4dad-bb05-8577c80b31f1\",\n      \"name\": \"3df7278a-fc9c-4dad-bb05-8577c80b31f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:55:02.189604085Z\",\n      \"endTime\": \"2026-10-07T18:19:08.046890686Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3aagkkq4l3ogcvf4h8o101v6q3m76 still exists (deletion dispatched at 2026-10-07T17:55:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3e339eaf0eb8\u003e: \n    resource group \"autoscaling-cluster-wq59b44864g7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wq59b44864g7\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/486d4d3b-fd8e-4b61-92c0-db85f958748b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/486d4d3b-fd8e-4b61-92c0-db85f958748b\",\n      \"name\": \"486d4d3b-fd8e-4b61-92c0-db85f958748b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:54:04.264847355Z\",\n      \"endTime\": \"2026-10-07T18:18:08.23090916Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta38okksp4b36cn28fmfj8oja6islo0 still exists (deletion dispatched at 2026-10-07T17:54:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3e339eaf0eb8\u003e: \n    resource group \"autoscaling-cluster-wq59b44864g7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wq59b44864g7\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/486d4d3b-fd8e-4b61-92c0-db85f958748b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/486d4d3b-fd8e-4b61-92c0-db85f958748b\",\n      \"name\": \"486d4d3b-fd8e-4b61-92c0-db85f958748b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:54:04.264847355Z\",\n      \"endTime\": \"2026-10-07T18:18:08.23090916Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta38okksp4b36cn28fmfj8oja6islo0 still exists (deletion dispatched at 2026-10-07T17:54:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x19e0b121c138\u003e: \n    resource group \"admin-api-serialconsole-8wkg2h2hv8wb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-8wkg2h2hv8wb\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52bbbc8d-3e28-45ff-86eb-4bc453001256\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52bbbc8d-3e28-45ff-86eb-4bc453001256\",\n      \"name\": \"52bbbc8d-3e28-45ff-86eb-4bc453001256\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:56:40.94108576Z\",\n      \"endTime\": \"2026-10-07T18:20:42.13673234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cm8fj69efgottp7couqt5o1bu2r2 still exists (deletion dispatched at 2026-10-07T17:56:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x19e0b121c138\u003e: \n    resource group \"admin-api-serialconsole-8wkg2h2hv8wb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-8wkg2h2hv8wb\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52bbbc8d-3e28-45ff-86eb-4bc453001256\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52bbbc8d-3e28-45ff-86eb-4bc453001256\",\n      \"name\": \"52bbbc8d-3e28-45ff-86eb-4bc453001256\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:56:40.94108576Z\",\n      \"endTime\": \"2026-10-07T18:20:42.13673234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cm8fj69efgottp7couqt5o1bu2r2 still exists (deletion dispatched at 2026-10-07T17:56:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"775e9e34036b536418ecb6ce7f646ee64060e4bac63d0b2703f2cb5c873f567c","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"a3363c6da7bde363b2cf3af699a41f3e3396a095df565f4aedc0f37273fedad8","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"9ee241ee8ad2e0f0ccaa82ff16aea7dd50b95471966e2fca705a2a6547db1ebe","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":3,"after_last_push_count":106,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":152,"prior_last_seen_at":"2026-09-30T18:13:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x9d4c3cd8090\u003e: \n    resource group \"admin-api-breakglass-6smbnkdj4hzs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-6smbnkdj4hzs\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ea14b34-3ced-4583-bf55-f2007e2d285f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ea14b34-3ced-4583-bf55-f2007e2d285f\",\n      \"name\": \"1ea14b34-3ced-4583-bf55-f2007e2d285f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:53:56.233010929Z\",\n      \"endTime\": \"2026-10-07T18:18:01.34812688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cjpfhsndlhr2qta8lp3hig1f25is still exists (deletion dispatched at 2026-10-07T17:53:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x9d4c3cd8090\u003e: \n    resource group \"admin-api-breakglass-6smbnkdj4hzs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-6smbnkdj4hzs\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ea14b34-3ced-4583-bf55-f2007e2d285f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ea14b34-3ced-4583-bf55-f2007e2d285f\",\n      \"name\": \"1ea14b34-3ced-4583-bf55-f2007e2d285f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:53:56.233010929Z\",\n      \"endTime\": \"2026-10-07T18:18:01.34812688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cjpfhsndlhr2qta8lp3hig1f25is still exists (deletion dispatched at 2026-10-07T17:53:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x25222e7880f0\u003e: \n    resource group \"rg-zstream-upgrade-4-20-42mlb9pj8467\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-2ns5vg\" in resourcegroup=\"rg-zstream-upgrade-4-20-42mlb9pj8467\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb6159e1-d013-447b-ad27-c209bc94526d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb6159e1-d013-447b-ad27-c209bc94526d\",\n      \"name\": \"bb6159e1-d013-447b-ad27-c209bc94526d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:52.899645021Z\",\n      \"endTime\": \"2026-10-07T18:25:01.15966395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cebl56c239c6lvk3po879sfe0nus still exists (deletion dispatched at 2026-10-07T18:00:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x25222e7880f0\u003e: \n    resource group \"rg-zstream-upgrade-4-20-42mlb9pj8467\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-2ns5vg\" in resourcegroup=\"rg-zstream-upgrade-4-20-42mlb9pj8467\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb6159e1-d013-447b-ad27-c209bc94526d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb6159e1-d013-447b-ad27-c209bc94526d\",\n      \"name\": \"bb6159e1-d013-447b-ad27-c209bc94526d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:52.899645021Z\",\n      \"endTime\": \"2026-10-07T18:25:01.15966395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cebl56c239c6lvk3po879sfe0nus still exists (deletion dispatched at 2026-10-07T18:00:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27dd6af64318\u003e: \n    resource group \"ingress-cert-le-h6vkqmnzj7lf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-h6vkqmnzj7lf\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6a648923-5c5a-40ca-8d8d-ae3a84c60046\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6a648923-5c5a-40ca-8d8d-ae3a84c60046\",\n      \"name\": \"6a648923-5c5a-40ca-8d8d-ae3a84c60046\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:01:35.349390537Z\",\n      \"endTime\": \"2026-10-07T18:25:41.362314296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cebs5l8pbehlv64f21cvh9tk83jo still exists (deletion dispatched at 2026-10-07T18:01:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27dd6af64318\u003e: \n    resource group \"ingress-cert-le-h6vkqmnzj7lf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-h6vkqmnzj7lf\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6a648923-5c5a-40ca-8d8d-ae3a84c60046\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6a648923-5c5a-40ca-8d8d-ae3a84c60046\",\n      \"name\": \"6a648923-5c5a-40ca-8d8d-ae3a84c60046\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:01:35.349390537Z\",\n      \"endTime\": \"2026-10-07T18:25:41.362314296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cebs5l8pbehlv64f21cvh9tk83jo still exists (deletion dispatched at 2026-10-07T18:01:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"004e0ec3888a99aaedcb7c2531a56de72dc32aa9559eb821f4fe7792ca5441d3","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"535c72aaa4c0665d8f64e9446dca3aa8e6d1fb91f52bd8fc1875bce44908d6ed","pr_number":7400,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"85866f3cd23f94e7a49f5e7cddfef735342e7f7a3e10ff85febdbe3d9afe74fe","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8af94ff77be9280ec3040c21d82445dd604d4b00fdf26e371d68448461bff48d","failure_pattern":"Command failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-\u003cid\u003e/pipeline@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/ci-op-\u003cid\u003e/pipeline:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-jhx30i3q/pipeline@sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 arohcpsvcdev.azurecr.io/ci-op-jhx30i3q/pipeline:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 --from-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json --to-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet/global/mirror-image","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-01T14:46:16Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet/global/mirror-image","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.KubeApplier/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-10-07T02:30:15.367Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e fleet.image.registry \u003cnil\u003e}:{\u003cnil\u003e fleet.image.repository \u003cnil\u003e}@{\u003cnil\u003e fleet.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-07T02:32:48.216Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build11.ci.openshift.org\\nSaved credentials for registry.build11.ci.openshift.org into /tmp/tmp.lx56KPpMDi/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build11.ci.openshift.org/ci-op-jhx30i3q/pipeline@sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 to arohcpsvcdev.azurecr.io/ci-op-jhx30i3q/pipeline:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604.\\nThe image will still be available under it's original digest sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 in the target registry.\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unavailable: service unavailable: health check failed: please see /debug/health\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unavailable: service unavailable: health check failed: please see /debug/health\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604: unknown: unknown error: map[DriverName:s3aws Enclosed:map[RequestFailure:map[]]]\\nCommand failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-jhx30i3q/pipeline@sha256:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 arohcpsvcdev.azurecr.io/ci-op-jhx30i3q/pipeline:a606d40bf095b5c6399b022598c612994fd9b0a54dccde130e796be7b44b7604 --from-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json --to-registry-config /tmp/tmp.lx56KPpMDi/containers/auth.json\\n exit status 1\"","time=2026-10-07T02:34:43.899Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.KubeApplier resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e kubeApplier.image.registry \u003cnil\u003e}:{\u003cnil\u003e kubeApplier.image.repository \u003cnil\u003e}@{\u003cnil\u003e kubeApplier.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-07T02:35:51.286Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.KubeApplier resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build11.ci.openshift.org\\nSaved credentials for registry.build11.ci.openshift.org into /tmp/tmp.BaWVNiNbL6/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build11.ci.openshift.org/ci-op-n30v64rp/pipeline@sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 to arohcpsvcdev.azurecr.io/ci-op-n30v64rp/pipeline:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894.\\nThe image will still be available under it's original digest sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 in the target registry.\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.build11.ci.openshift.org/v2/ci-op-n30v64rp/pipeline/manifests/sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894\\\": response status code 503: Service Unavailable\\nCommand failed after 5 attempts: oras cp registry.build11.ci.openshift.org/ci-op-n30v64rp/pipeline@sha256:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 arohcpsvcdev.azurecr.io/ci-op-n30v64rp/pipeline:9703abd2b587ea12d843258574bb8f17d0cee4bd68cb43fcbd7116f6e22ea894 --from-registry-config /tmp/tmp.BaWVNiNbL6/containers/auth.json --to-registry-config /tmp/tmp.BaWVNiNbL6/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647565049630720","occurred_at":"2026-10-07T01:41:52Z","signature_id":"2d70df3f3655f8e6d61803842ff1a711d8b43be234f53bc324901afc6271aab3","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647281405628416","occurred_at":"2026-10-07T01:40:44Z","signature_id":"16290eb32430f166cec00354029ad2ead97a9b7e00cd7ae123762fc3b2a7ef4a","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-kcpxjfv4fpw8\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-30T06:47:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x213e82b661e0\u003e: \n    resource group \"rg-cluster-back-version-s9llrx7qtffw\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x213e826ac820\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-s9llrx7qtffw\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x213e826ac800\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x213e83e111d0\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x213e82b661e0\u003e: \n    resource group \"rg-cluster-back-version-s9llrx7qtffw\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-s9llrx7qtffw\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x213e826ac820\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-s9llrx7qtffw\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x213e826ac800\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x213e83e111d0\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x213e83e11170\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-s9llrx7qtffw\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x213e831e7270\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-s9llrx7qtffw\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3c28962ca078\u003e: \n    resource group \"rg-cluster-back-version-kcpxjfv4fpw8\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3c289468e340\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-kcpxjfv4fpw8\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x3c289468e320\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x3c2894a68660\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3c28962ca078\u003e: \n    resource group \"rg-cluster-back-version-kcpxjfv4fpw8\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kcpxjfv4fpw8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3c289468e340\u003e{\n                msg: \"resource group \\\"rg-cluster-back-version-kcpxjfv4fpw8\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                err: \u003c*fmt.wrapError | 0x3c289468e320\u003e{\n                    msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                    err: \u003c*fmt.wrapErrors | 0x3c2894a68660\u003e{\n                        msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                        errs: [\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                            \u003c*fmt.wrapErrors | 0x3c2894a68630\u003e{\n                                msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kcpxjfv4fpw8\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8, error: context deadline exceeded\",\n                                errs: [\n                                    \u003c*errors.errorString | 0x3c2894fa1480\u003e{\n                                        s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kcpxjfv4fpw8\",\n                                    },\n                                    \u003ccontext.deadlineExceededError\u003e{},\n                                ],\n                            },\n                        ],\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"4d8dc6809a1434d6bdd49c263f770a407adc719dc2f0c6e5c6248a598c1ba4ad","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"ad853bd0242f92cb0e2af68ec14f11bc3c9f69d37f2035d60a8204f74aa62438","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x295f3df44be0\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-f56729plnfx5\": PUT https://rp.j6769152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-f56729plnfx5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x295f3df44be0\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-f56729plnfx5\": PUT https://rp.j6769152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-f56729plnfx5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x156cf26d7900\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-j4cvjbg6jcxh\": PUT https://rp.j0287232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j4cvjbg6jcxh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n    \u003c*fmt.wrapError | 0x156cf26d7900\u003e: \n    failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-j4cvjbg6jcxh\": PUT https://rp.j0287232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j4cvjbg6jcxh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Too long: may not be more than 15 bytes\",\n            \"target\": \"id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n            \"target\": \"id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"b243e7b21db9c6849fd992f8e4574d0ab72105b36c17e4d86b14eba547a70a25","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107681070840287232","occurred_at":"2026-10-07T03:55:00Z","signature_id":"80e9dbbbafdd3fbd1f829013bf1ed0880b1a13d485e1a90cecc503401b7f6c2d","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f9989e753db1daf9717992966720fb7c20aebb56a0a3e0abe95adf30bb4255a7","failure_pattern":"Process did not finish before 15m0s timeout","search_query":"Process did not finish before 15m0s timeout","failed_at":"unknown","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-22T22:46:36Z","contributing_tests":[{"failed_at":"unknown","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:178\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 15m0s timeout\",\"severity\":\"error\",\"time\":\"2026-10-07T15:54:40Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:178\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 15m0s timeout\",\"severity\":\"error\",\"time\":\"2026-10-07T06:30:56Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"20a552c9ef65c7031e4db49271edb6fa7f672d4bde424eea2c61b746d13edef7","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"20a552c9ef65c7031e4db49271edb6fa7f672d4bde424eea2c61b746d13edef7","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-30T16:03:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","occurrences":2}],"full_error_samples":["alert fired 8 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T14:28:55Z\n  Ended:       2026-10-07T15:55:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vtqjq0sahnttfr0obenao468e4qlb-e0e2j3m7x8c7o1b\", region=\"westus3\", severity=\"warning\"\n  Description: 10 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vtqjq0sahnttfr0obenao468e4qlb-e0e2j3m7x8c7o1b (management cluster ci01-j9974528-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:27:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vuf7r0mkidc4qe1opuvsfmq0jlia8-p9j3q3c5l3l9j9t\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vuf7r0mkidc4qe1opuvsfmq0jlia8-p9j3q3c5l3l9j9t (management cluster ci01-j9974528-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:27:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vu5443vh0o9q88a97n0snmv6qq2pb-acr-pull\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vu5443vh0o9q88a97n0snmv6qq2pb-acr-pull (management cluster ci01-j9974528-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:27:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vu0gdmpn5k5d5bgvk4606cnb2ql44-s5l1e0o2a6j6m8n\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vu0gdmpn5k5d5bgvk4606cnb2ql44-s5l1e0o2a6j6m8n (management cluster ci01-j9974528-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-07T15:27:53Z\n  Ended:       2026-10-07T15:46:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vtnemv5o1grorom2pjgo9il7l5cgq-l7p8p3x8g9o9l5c\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vtnemv5o1grorom2pjgo9il7l5cgq-l7p8p3x8g9o9l5c (management cluster ci01-j9974528-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:27:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vtt5ll44f8b35n9lm1o4n0oims33s-private-kas\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vtt5ll44f8b35n9lm1o4n0oims33s-private-kas (management cluster ci01-j9974528-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-07T15:27:53Z\n  Ended:       2026-10-07T15:47:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vtu5p75nk86f54m2uds6vreosd2p2-v0n4p7w8h2f0o9a\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vtu5p75nk86f54m2uds6vreosd2p2-v0n4p7w8h2f0o9a (management cluster ci01-j9974528-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-07T15:27:53Z\n  Ended:       2026-10-07T15:47:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j9974528-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9vu3ea2tjsap1sokp49bjsntgc3llb-u9p1t5k3r2x5x0n\", region=\"westus3\", severity=\"warning\"\n  Description: 8 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9vu3ea2tjsap1sokp49bjsntgc3llb-u9p1t5k3r2x5x0n (management cluster ci01-j9974528-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T05:10:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nlcah4jgmcp45v4eq4gnjhrn1t86e-ingress-cert-le\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nlcah4jgmcp45v4eq4gnjhrn1t86e-ingress-cert-le (management cluster ci01-j2737280-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-07T06:04:06Z\n  Ended:       2026-10-07T06:30:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nlr9uea4a002u1vobs1eamhd9ij0g-l5p7g4y6r1u6h3x\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nlr9uea4a002u1vobs1eamhd9ij0g-l5p7g4y6r1u6h3x (management cluster ci01-j2737280-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-07T06:04:06Z\n  Ended:       2026-10-07T06:29:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nldv9i221aq0hikhv2urfttr6p4ag-j3v3v2v9p0l8t7b\", region=\"westus3\", severity=\"warning\"\n  Description: 8 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nldv9i221aq0hikhv2urfttr6p4ag-j3v3v2v9p0l8t7b (management cluster ci01-j2737280-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T06:04:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nlhakelm48kkcoo98ifgh0s48qjr9-x7k8b4r2l9a9q2i\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nlhakelm48kkcoo98ifgh0s48qjr9-x7k8b4r2l9a9q2i (management cluster ci01-j2737280-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-07T06:04:06Z\n  Ended:       2026-10-07T06:29:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nleqkohnk6u75anrikd0ihfs6jmk6-i8v1g1d3d9x0t2s\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nleqkohnk6u75anrikd0ihfs6jmk6-i8v1g1d3d9x0t2s (management cluster ci01-j2737280-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-07T06:04:06Z\n  Ended:       2026-10-07T06:32:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nllqoph4r7lp0chrtmvsdfajj63mb-s0d9l7t2s9f2t9s\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nllqoph4r7lp0chrtmvsdfajj63mb-s0d9l7t2s9f2t9s (management cluster ci01-j2737280-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-07T06:04:06Z\n  Ended:       2026-10-07T06:30:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nldc2p59tvtq108i7fn6ktamnpfd9-private-kas\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nldc2p59tvtq108i7fn6ktamnpfd9-private-kas (management cluster ci01-j2737280-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.\n\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-07T06:04:06Z\n  Ended:       2026-10-07T06:33:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2737280-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t9nlp25sdnfj4rsjsnnajlljlta2m49-acr-pull\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t9nlp25sdnfj4rsjsnnajlljlta2m49-acr-pull (management cluster ci01-j2737280-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"9fb80e317ba17a110379c8169e33f721a9e1b46fd5cbd72a1477ce7cdef9b69a","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"947f42338d6e84abb9bb52e5b52d433427544ab1231ca5eddc6f0f75313afdfc","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":20,"trend_counts":[5,1,0,0,7,7,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":62,"prior_last_seen_at":"2026-10-02T10:59:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T18:34:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8893056-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.61:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8893056/providers/microsoft.monitor/accounts/hcps-j8893056\", namespace=\"ocm-arohcpci01-2ta3f94m6csa7tur24re6meedru3s5o4-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-n8krx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ta3f94m6csa7tur24re6meedru3s5o4-ksm-hcp-metrics/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T18:34:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8893056-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.61:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8893056/providers/microsoft.monitor/accounts/hcps-j8893056\", namespace=\"ocm-arohcpci01-2ta3f94m6csa7tur24re6meedru3s5o4-ksm-hcp-metrics\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-n8krx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ta3f94m6csa7tur24re6meedru3s5o4-ksm-hcp-metrics/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T18:29:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3305728-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3305728/providers/microsoft.monitor/accounts/hcps-j3305728\", namespace=\"ocm-arohcpci01-2ta38bnq43hvrce83gic7njl84d30rep-i1r8s9j1s0i1f3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gmh4h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ta38bnq43hvrce83gic7njl84d30rep-i1r8s9j1s0i1f3t/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T18:29:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3305728-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3305728/providers/microsoft.monitor/accounts/hcps-j3305728\", namespace=\"ocm-arohcpci01-2ta38bnq43hvrce83gic7njl84d30rep-i1r8s9j1s0i1f3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gmh4h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ta38bnq43hvrce83gic7njl84d30rep-i1r8s9j1s0i1f3t/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"10d4a1a04107eb655251aa6f4caa6a71839828780a3edffbdda858f07d1855cc","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"de6beb6bf4334e9b143d9dd4dbc06e85520a4d8bbe278fddc25171d8b71278b4","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-03T03:24:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T14:03:14Z\n  Ended:       2026-10-07T14:20:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9974528-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T04:40:19Z\n  Ended:       2026-10-07T04:55:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2737280-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"125a20f06d40b81d0b44090ea3789c5fe4521773a1cc660485637fe5bfa1afbb","pr_number":7407,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"125a20f06d40b81d0b44090ea3789c5fe4521773a1cc660485637fe5bfa1afbb","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":9,"trend_counts":[0,3,1,0,4,5,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":54,"prior_last_seen_at":"2026-10-03T00:45:21Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:02:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-c4d6b786-bgkk8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-c4d6b786-bgkk8 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T02:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5628416-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-445sp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-445sp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T02:43:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5628416-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-gpz7w\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-gpz7w has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107961335642853376","occurred_at":"2026-10-07T22:28:41Z","signature_id":"f3afea088c71927d2827ae8a8594776068254cc9904aa88bdae7db71ce3d503a","pr_number":7305,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647281405628416","occurred_at":"2026-10-07T01:40:44Z","signature_id":"9ee55a303c8e841323cc685a7ec3f29ef79b76512b54804abd0bdb38ccfc1218","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"da6ef9b7d4b10235ac25ede586d99e51451945ec64f5760ee4de6b9d47ea200e","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed starting nodepool creation \"np-1\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\": PUT \u003curl\u003e","search_query":"failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-fh65w65xjrlv\": PUT https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e182923c240\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ls445v45mtp4\": PUT https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-ls445v45mtp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e182923c240\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-ls445v45mtp4\": PUT https://rp.j9599616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-ls445v45mtp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:162]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e9eea24a6c0\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-fh65w65xjrlv\": PUT https://rp.j8091008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-fh65w65xjrlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:162]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2e9eea24a6c0\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-fh65w65xjrlv\": PUT https://rp.j8091008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-fh65w65xjrlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21.36\\\": node pool version 4.21.36 cannot coexist with control plane version 5.1.0-ec.1\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"3fd8a931a3e5bf9db95183328d35b726f2c6c6c4e8ee953ac0421be7cfbc1384","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107817608098091008","occurred_at":"2026-10-07T12:57:33Z","signature_id":"6865a4e5afd5f302e897a1e9b8a117f3650268c89570c1a3a5cb4b887706e488","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-d5s257gp6qv6\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.816901408450704,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-29T11:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x50103f9c9c0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-d5s257gp6qv6\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-d5s257gp6qv6\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x50103f9c9c0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-d5s257gp6qv6\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-d5s257gp6qv6\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c581ef3e4a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wcbt6xkttfqm\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-wcbt6xkttfqm\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c581ef3e4a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wcbt6xkttfqm\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-20-32\\\" for cluster \\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-wcbt6xkttfqm\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107841500657225728","occurred_at":"2026-10-07T14:32:30Z","signature_id":"fec7daaa523868402b6b536ad9a8369cf346299cb7d8ea871ac465218dc8895b","pr_number":6954,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107828301232869376","occurred_at":"2026-10-07T13:40:03Z","signature_id":"420046f20ff960ccdda2a598e04cfcbca82502afdbf5b94e7a99272cdc27ca96","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8408b249004f1eb1294f56285dbfbec0fecc715e2e28bc3573e8af1426314384","failure_pattern":"ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])","search_query":"ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to see an HCP cluster registered with console.redhat.com via the Insights Operator","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:142]: Timed out after 1200.007s.\nInsights Operator did not reach Disabled/NoToken state, the NoToken-to-healthy transition cannot be tested\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x27811265a7d0\u003e: \n    ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])\n    {\n        s: \"ClusterOperator \\\"insights\\\" not yet in Disabled/NoToken state (conditions: [])\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:142]: Timed out after 1200.007s.\nInsights Operator did not reach Disabled/NoToken state, the NoToken-to-healthy transition cannot be tested\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x27811265a7d0\u003e: \n    ClusterOperator \"insights\" not yet in Disabled/NoToken state (conditions: [])\n    {\n        s: \"ClusterOperator \\\"insights\\\" not yet in Disabled/NoToken state (conditions: [])\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107939651057618944","occurred_at":"2026-10-07T21:02:31Z","signature_id":"748b7886a08fb3fd1cc9a8d22e3de74573c8fbb8f8d74801689482e69f7cff3c","pr_number":6954,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-29T01:37:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x219764076460\u003e: \n    DNS for route host agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x219764076440\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x219764076460\u003e: \n    DNS for route host agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x219764076440\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2bp46.apps.aro.l8a4l4n9c3b5o6z.me9l.j6769152.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107751278686769152","occurred_at":"2026-10-07T08:33:59Z","signature_id":"b2b88a83521cee771cd93ba8e8c7482cb474d79b4714e2d4c669fa1ea2c77764","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,2,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f1b762f1400\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-dzl9ch5fblqj\" to finish creating: GET https://rp.j2675200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/597912cd-0173-4a02-a3e6-da94ac3b74b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/597912cd-0173-4a02-a3e6-da94ac3b74b6\",\n      \"name\": \"597912cd-0173-4a02-a3e6-da94ac3b74b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:41:10.05474314Z\",\n      \"endTime\": \"2026-10-07T18:10:19.11453378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1f1b762f1400\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-dzl9ch5fblqj\" to finish creating: GET https://rp.j2675200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/597912cd-0173-4a02-a3e6-da94ac3b74b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/597912cd-0173-4a02-a3e6-da94ac3b74b6\",\n      \"name\": \"597912cd-0173-4a02-a3e6-da94ac3b74b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:41:10.05474314Z\",\n      \"endTime\": \"2026-10-07T18:10:19.11453378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107872947002675200","occurred_at":"2026-10-07T16:37:27Z","signature_id":"6aa7421e150db2feff9236efdb495765f16c29a20bce67118d40673f019a35dc","pr_number":7310,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2c785268dead82f5391d67011a5ed778899c269360f3bc0b5673c6c7eb09c779","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; hypershift NodePool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; hypershift NodePool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0b67484b00\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-b6bhhgvdr2ld\" to finish updating: GET https://rp.j0967424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"name\": \"9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:15:51.4879876Z\",\n      \"endTime\": \"2026-10-07T18:34:57.661146093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool status replicas is 2, want 3; hypershift NodePool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0b67484b00\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-b6bhhgvdr2ld\" to finish updating: GET https://rp.j0967424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"name\": \"9d8af708-d204-41c1-8bc5-4f60976f86b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:15:51.4879876Z\",\n      \"endTime\": \"2026-10-07T18:34:57.661146093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool status replicas is 2, want 3; hypershift NodePool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; hypershift NodePool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine s0q3e0l7h6y3n3h-np-1-njcq9-gnl4f: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7322/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875761300967424","occurred_at":"2026-10-07T16:48:38Z","signature_id":"3eddb723e07465e8928579f25de472d49a61e72fa22ed9364fe0258841f58a49","pr_number":7322,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"03191c7ee6ef10f05815abde7f8b6f6bad2b4b3b1acdea0c834f5ddce823bc91","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3974085580d8\u003e: \n    resource group \"external-auth-cluster-jkpp8mdvzxsh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jkpp8mdvzxsh\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"name\": \"7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:50.350274496Z\",\n      \"endTime\": \"2026-10-07T18:24:57.82988818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3adt66dtiffarpvt6smilc2f19037 still exists (deletion dispatched at 2026-10-07T18:00:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x3974085580d8\u003e: \n    resource group \"external-auth-cluster-jkpp8mdvzxsh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jkpp8mdvzxsh\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"name\": \"7ef42875-c74c-425c-8fe8-0909778dceac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:50.350274496Z\",\n      \"endTime\": \"2026-10-07T18:24:57.82988818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3adt66dtiffarpvt6smilc2f19037 still exists (deletion dispatched at 2026-10-07T18:00:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"ad65d918080739de6f2f3b2f425a73e3b32c8cc57bfe3363cce8fe1a1dc681fd","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0cce093122447f5c3b6f99be8e0de298d1fbee72db77c1703737ae037ec0882b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2183e1e34210\u003e: \n    resource group \"private-ingress-76lkp46mpc8d\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-76lkp46mpc8d\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"name\": \"1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:03:40.469898232Z\",\n      \"endTime\": \"2026-10-07T18:27:41.300163086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cd0prbd7pkt4n50qh2rsnhp0bjo4 still exists (deletion dispatched at 2026-10-07T18:03:41Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2183e1e34210\u003e: \n    resource group \"private-ingress-76lkp46mpc8d\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-76lkp46mpc8d\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"name\": \"1f6732c1-e698-4577-a105-6f4bedb6c48b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:03:40.469898232Z\",\n      \"endTime\": \"2026-10-07T18:27:41.300163086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3cd0prbd7pkt4n50qh2rsnhp0bjo4 still exists (deletion dispatched at 2026-10-07T18:03:41Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"ecbdfd8493553d96ba4723a52e827625dceadf0a4d4fcf8208229496ffbca565","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3f397df201c8\u003e: \n    resource group \"clusternp128-89h59bcbsmnq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-89h59bcbsmnq\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"name\": \"84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:09.863415259Z\",\n      \"endTime\": \"2026-10-07T18:24:11.307794702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3chch61ipf94injnrljj1hk0caf92 still exists (deletion dispatched at 2026-10-07T18:00:10Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3f397df201c8\u003e: \n    resource group \"clusternp128-89h59bcbsmnq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-89h59bcbsmnq\" to finish deleting: GET https://rp.j6658560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"name\": \"84ac2f7c-19a8-4b2d-82b9-6595d5a024a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:00:09.863415259Z\",\n      \"endTime\": \"2026-10-07T18:24:11.307794702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ta3chch61ipf94injnrljj1hk0caf92 still exists (deletion dispatched at 2026-10-07T18:00:10Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875420446658560","occurred_at":"2026-10-07T16:47:17Z","signature_id":"cc042585e1ff82e3d4f824868e1321ec45e4977ce868ac923076ba255c0d42a1","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0xeda890f47f8\u003e: \n    resource group \"oidc-wi-h74rjsb4b6b4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-h74rjsb4b6b4\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/347d2fde-a5b9-4245-8439-df8aaab88f98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/347d2fde-a5b9-4245-8439-df8aaab88f98\",\n      \"name\": \"347d2fde-a5b9-4245-8439-df8aaab88f98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:11:44.745635305Z\",\n      \"endTime\": \"2026-10-07T18:35:51.763179185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0xeda890f47f8\u003e: \n    resource group \"oidc-wi-h74rjsb4b6b4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-h74rjsb4b6b4\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/347d2fde-a5b9-4245-8439-df8aaab88f98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/347d2fde-a5b9-4245-8439-df8aaab88f98\",\n      \"name\": \"347d2fde-a5b9-4245-8439-df8aaab88f98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:11:44.745635305Z\",\n      \"endTime\": \"2026-10-07T18:35:51.763179185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"8c20bdbff176792aa32d1f70b7055efc5adcd7e98259606ec8b663cd51ef5fe0","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-22T22:46:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3ed0c46420d8\u003e: \n    resource group \"rg-negative-tests-z8rjxzl5wzq5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-z8rjxzl5wzq5\" to finish deleting: GET https://rp.j8408192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"name\": \"92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:31:10.664805555Z\",\n      \"endTime\": \"2026-10-07T20:55:12.515323239Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x3ed0c46420d8\u003e: \n    resource group \"rg-negative-tests-z8rjxzl5wzq5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-z8rjxzl5wzq5\" to finish deleting: GET https://rp.j8408192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"name\": \"92268962-7674-4c53-a6ec-a9b7d3b7b795\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:31:10.664805555Z\",\n      \"endTime\": \"2026-10-07T20:55:12.515323239Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"0ecb42980a83f2bed43b3f38c65950d796ceed1be274e4dbc2e1013fe3707753","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-17T08:16:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x1df0828207f8\u003e: \n    resource group \"rg-zstream-upgrade-4-23-4xq25ksc85ts\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-23-vhm2c5\" in resourcegroup=\"rg-zstream-upgrade-4-23-4xq25ksc85ts\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f203186-8575-4f16-9fec-af4db81dcdaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f203186-8575-4f16-9fec-af4db81dcdaf\",\n      \"name\": \"7f203186-8575-4f16-9fec-af4db81dcdaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:02:18.08747435Z\",\n      \"endTime\": \"2026-10-07T18:26:27.625877529Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n    \u003c*errors.joinError | 0x1df0828207f8\u003e: \n    resource group \"rg-zstream-upgrade-4-23-4xq25ksc85ts\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-23-vhm2c5\" in resourcegroup=\"rg-zstream-upgrade-4-23-4xq25ksc85ts\" to finish deleting: GET https://rp.j3305728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f203186-8575-4f16-9fec-af4db81dcdaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f203186-8575-4f16-9fec-af4db81dcdaf\",\n      \"name\": \"7f203186-8575-4f16-9fec-af4db81dcdaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:02:18.08747435Z\",\n      \"endTime\": \"2026-10-07T18:26:27.625877529Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107875990943305728","occurred_at":"2026-10-07T16:49:33Z","signature_id":"df2ee66aadf49a7254b9c095a9ada5b4b3e24662144e3c9aaa040cca41949609","pr_number":7143,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db15661ea057fcdb86696aa934e0ebb9321ec309068c024420faf52e725ffb06","failure_pattern":"ERROR: Please run 'az login' to setup account.","search_query":"ERROR: Please run 'az login' to setup account.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T09:08:54Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-07T09:04:20.332Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-10-07T09:04:22.138Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nERROR: Please run 'az login' to setup account.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750698572582912","occurred_at":"2026-10-07T08:31:41Z","signature_id":"21658b9b3d6cf9bf31715ba32c333953291d0d0fe4a6e4e41252f24603e05dff","pr_number":7398,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1ae9bcc864e0b95b17cfb70f07fc859ef6b54ed4d13d4886751e4ae7fe6a17c0","failure_pattern":"Internal error occurred: failed calling webhook \"webhook","search_query":"Internal error occurred: failed calling webhook \"webhook","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:337]: Timed out after 120.000s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x3dfc585d0640\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:337]: Timed out after 120.000s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0x3dfc585d0640\u003e: \n    Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        ErrStatus: {\n            TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n            ListMeta: {\n                SelfLink: \"\",\n                ResourceVersion: \"\",\n                Continue: \"\",\n                RemainingItemCount: nil,\n                ShardInfo: nil,\n            },\n            Status: \"Failure\",\n            Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n            Reason: \"InternalError\",\n            Details: {\n                Name: \"\",\n                Group: \"\",\n                Kind: \"\",\n                UID: \"\",\n                Causes: [\n                    {\n                        Type: \"\",\n                        Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        Field: \"\",\n                    },\n                ],\n                RetryAfterSeconds: 0,\n            },\n            Code: 500,\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107750168211230720","occurred_at":"2026-10-07T08:29:34Z","signature_id":"aa5a79bb72cb1d70468a298f56bdb83213121c034823482ee690c746113359f8","pr_number":7370,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:128]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0x3fa9a7564d98\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3fa9a6cff3e0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6cff3c0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a7398db0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0x3fa9a716a690\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3fa9a6ee5c60\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6ee5c40\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a786a420\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0x3fa9a7c7d0e0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:128]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0x3fa9a7564d98\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x3fa9a6cff3e0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6cff3c0\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a7398db0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0x3fa9a716a690\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x3fa9a6ee5c60\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x3fa9a6ee5c40\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x3fa9a786a420\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0x3fa9a7c7d0e0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x3fa9a7c7cfa0\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.157.3:34489-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.g0o4h9l1p1i8t8c.5oub.j2737280.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7400/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107676117312737280","occurred_at":"2026-10-07T03:35:19Z","signature_id":"9f080afeacc6660909212d05408fd022c6a74fba0cb0a9759c7ec00f6ad7192d","pr_number":7400,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1b8edd8bf3beb856400cfdf84db0af55a47389abcf6ab46ec6999548fb8b8f61","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-tkqcj)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xf6f46a980d8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-tkqcj)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xf6f45174120\u003e{\n                msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                err: \u003c*fmt.wrapError | 0xf6f45174100\u003e{\n                    msg: \"failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                    err: \u003c*fmt.wrapError | 0xf6f451742a0\u003e{\n                        msg: \"failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                        err: \u003c*errors.StatusError | 0xf6f4458f7c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                                Reason: \"InternalError\",\n                                Details: {\n                                    Name: \"fips-check-tkqcj\",\n                                    Group: \"\",\n                                    Kind: \"pods\",\n                                    UID: \"\",\n                                    Causes: [\n                                        {Type: ..., Message: ..., Field: ...},\n                                    ],\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 500,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xf6f46a980d8\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-tkqcj)\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xf6f45174120\u003e{\n                msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                err: \u003c*fmt.wrapError | 0xf6f45174100\u003e{\n                    msg: \"failed to check FIPS mode on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p: failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                    err: \u003c*fmt.wrapError | 0xf6f451742a0\u003e{\n                        msg: \"failed to get pod logs for pod fips-check-tkqcj in namespace e2e-fips-test-zn5bh on node h1f2m2p1y3c2t8a-np-1-5mzcd-8fr8p after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                        err: \u003c*errors.StatusError | 0xf6f4458f7c0\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-tkqcj)\",\n                                Reason: \"InternalError\",\n                                Details: {\n                                    Name: \"fips-check-tkqcj\",\n                                    Group: \"\",\n                                    Kind: \"pods\",\n                                    UID: \"\",\n                                    Causes: [\n                                        {Type: ..., Message: ..., Field: ...},\n                                    ],\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 500,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107916955439599616","occurred_at":"2026-10-07T19:32:18Z","signature_id":"b2817992361bcd87227b455397537c7fa2c5d587900b532a6d346247a6380362","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x5336d4f4280\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x5336d4f4220\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x5336d4f4180\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x5336d4f4280\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.008s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x5336d4f4220\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x5336d4f4180\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107970662109810688","occurred_at":"2026-10-07T23:05:44Z","signature_id":"9af334222a9a1079a318e8007182007684788c07d1bd75371b6aa376ab0cd4f3","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2ea683689d213a80fd9693db9b372fae798477a56e7e530874ac1f5a4eddf2fd","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-28T16:12:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d103f80a620\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        err: \u003c*errors.errorString | 0x1d103f3186b0\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d103f80a620\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        err: \u003c*errors.errorString | 0x1d103f3186b0\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\\n  - pod pull-secret-test container pull-test: ContainerCreating ()\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107817608098091008","occurred_at":"2026-10-07T12:57:33Z","signature_id":"7a2c94fb45e7db0296dcd9f041aa0c8c5dcbc38fae73401939da42d841d4b069","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":14,"trend_counts":[5,1,0,0,2,5,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":35,"prior_last_seen_at":"2026-10-02T10:59:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T18:21:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8893056-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.61:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8893056/providers/microsoft.monitor/accounts/hcps-j8893056\", namespace=\"ocm-arohcpci01-2ta3f94m6csa7tur24re6meedru3s5o4-ksm-hcp-metrics\", pod=\"cloud-network-config-controller-8445d77f54-dbcgg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b63493d6-07ef-4acd-8041-60edf966639a\"\n  Description: Pod ocm-arohcpci01-2ta3f94m6csa7tur24re6meedru3s5o4-ksm-hcp-metrics/cloud-network-config-controller-8445d77f54-dbcgg (controller) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107879778638893056","occurred_at":"2026-10-07T17:04:36Z","signature_id":"93daf971eb2e3d807bd5be43c5947b64fbc568467906d318a73562ccf23adc89","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":23,"prior_last_seen_at":"2026-09-30T05:18:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 9 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:27:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:27:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-07T23:34:27Z\n  Ended:       2026-10-07T23:43:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta829bp1fhgrfupabf34hjhbao8sjia\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta829bp1fhgrfupabf34hjhbao8sjia/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-07T23:35:31Z\n  Ended:       2026-10-07T23:45:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta82bca3t1j71tbsbphnabaj9385rs2\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta82bca3t1j71tbsbphnabaj9385rs2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-07T23:35:32Z\n  Ended:       2026-10-07T23:54:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-07T23:59:27Z\n  Ended:       2026-10-08T00:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8e00ategjs6jdt7dm9iouhjbvp29q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8e00ategjs6jdt7dm9iouhjbvp29q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-08T00:00:27Z\n  Ended:       2026-10-08T00:10:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8erefr60nd4djngqkit7pco8mj14u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8erefr60nd4djngqkit7pco8mj14u/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-08T00:00:27Z\n  Ended:       2026-10-08T00:07:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8f7d8ssfok4nfel8v6oomstor9osq\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8f7d8ssfok4nfel8v6oomstor9osq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-08T00:02:28Z\n  Ended:       2026-10-08T00:14:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta8fb8j9edjipdiide2u4g57sv913hs\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ta8fb8j9edjipdiide2u4g57sv913hs/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107961335642853376","occurred_at":"2026-10-07T22:28:41Z","signature_id":"d5d12fa7c02227280374e42810634b4c5d7062a37a2bf81274486c206da8d33b","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-30T05:18:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T23:37:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-07T23:45:38Z\n  Ended:       2026-10-07T23:54:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ta82divi7qc3dtot56dc9nrv2029osp/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-08T00:09:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j2853376-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2853376/providers/microsoft.monitor/accounts/services-j2853376\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7t5pm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107961335642853376","occurred_at":"2026-10-07T22:28:41Z","signature_id":"ae24b3bd209fd3bdae001beee3f0cc45e8c82e783e565d4b2501a33d06dca5b5","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,2,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-25T10:20:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-07T02:35:12Z\n  Ended:       2026-10-07T02:42:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j5628416-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107647281405628416","occurred_at":"2026-10-07T01:40:44Z","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":44,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":109,"prior_last_seen_at":"2026-09-30T06:47:34Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-07T15:26:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9974528-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jrr5z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-jrr5z (namespace arobit) on cluster ci01-j9974528-svc has averaged 124.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7407/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107818144989974528","occurred_at":"2026-10-07T12:59:41Z","signature_id":"8a69335c6b425dfcfab867320cdd5348fb00e994d8030c8247190937dcfb5c68","pr_number":7407,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":10,"also_in":["PROD"],"trend_counts":[1,1,0,0,1,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":22,"prior_last_seen_at":"2026-10-02T18:39:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e187b528690\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e187b528690\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107680144293367808","occurred_at":"2026-10-07T03:51:19Z","signature_id":"d1dfb5d720d4e924abf2ec1c610faee1a9a11cd6f9af55462d2c0c27633e01df","pr_number":7097,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3bc6a7df03bfd8951b086726c7bf177263d62d8ccd5624cb33a8a5e6706353f6","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x4aa5231ef20\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\n    let _endTime = datetime(2026-10-07T20:13:47.1720653Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-gg7xds6svpzk'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T20:13:54.1282773Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\", ActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", SubActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityStack:\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x4aa53d27068\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x4aa5231ef00\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x4aa5231eec0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x4aa52207ce0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x4aa52207ca0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x4aa52820300\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x4aa5231ef20\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\n    let _endTime = datetime(2026-10-07T20:13:47.1720653Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-gg7xds6svpzk'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T20:13:54.1282773Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\", ActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", SubActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f024af6-e2e7-4ff7-91db-bc5319766d55\", ActivityStack:\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x4aa53d27068\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x4aa5231ef00\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x4aa5231eec0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x4aa52207ce0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x4aa52207ca0\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T20:13:47.1720651Z);\\nlet _endTime = datetime(2026-10-07T20:13:47.1720653Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-gg7xds6svpzk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T20:13:54.1282773Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77\\\", ActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", SubActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"9f024af6-e2e7-4ff7-91db-bc5319766d55\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f1a232d6-957a-4fb2-b729-3a7bbce9fa77 ARID=9f024af6-e2e7-4ff7-91db-bc5319766d55 \u003e GW.Http.CallContext/9f024af6-e2e7-4ff7-91db-bc5319766d55)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x4aa52820300\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904235558408192","occurred_at":"2026-10-07T18:41:47Z","signature_id":"7d412545a42210e54d12fe8199b4fe8b2c91f6c65dc9ad7a4d53e143213f22db","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c7241dd3402b660f1519d58411034f82666a2c1c7f982d027aba3c2a37a8cdb1","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x339d2c183220\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\n    let _endTime = datetime(2026-10-07T22:32:32.2954057Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T22:32:43.6043688Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\", ActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", SubActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x339d2b3c97a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x339d2c183200\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x339d2c1831e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x339d2c1830c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x339d2c183080\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x339d2ae3d200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x339d2c183220\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\n    let _endTime = datetime(2026-10-07T22:32:32.2954057Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-07T22:32:43.6043688Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\", ActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", SubActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\", ActivityStack:\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x339d2b3c97a0\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x339d2c183200\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x339d2c1831e0\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x339d2c1830c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x339d2c183080\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-06T22:32:32.2954055Z);\\nlet _endTime = datetime(2026-10-07T22:32:32.2954057Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-n8d6fgpt4qrq'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-07T22:32:43.6043688Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd\\\", ActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", SubActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"135bb6a2-6f7b-4d3f-a01b-32572cd9031c\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;ba806fbf-42fd-4d1f-9a02-0f02e32bcdfd ARID=135bb6a2-6f7b-4d3f-a01b-32572cd9031c \u003e GW.Http.CallContext/135bb6a2-6f7b-4d3f-a01b-32572cd9031c)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x339d2ae3d200\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6954/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107939651057618944","occurred_at":"2026-10-07T21:02:31Z","signature_id":"707d524bc3708f6d8c9654e45c0eaa4ed86159ce6bbac3cc9b6a73ce91708f8f","pr_number":6954,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"172a674b0a38bb10e2125b5a2867d55ded44654f7e5e154155ab760de9601ea3","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11eea31d1d40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\n    let _endTime = datetime(2026-10-08T00:23:30.3337686Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-7nt846zks6zp'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T00:23:37.3079994Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\", ActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", SubActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityStack:\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x11eea2518d38\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x11eea31d1d20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x11eea31d1d00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x11eea2cd05c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x11eea2cd0560\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x11eea1622000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x11eea31d1d40\u003e: \n    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\n    let _endTime = datetime(2026-10-08T00:23:30.3337686Z);\n    // Extract cluster ID from CS logs using cid field\n    let _cs_cluster_id = toscalar(\n        database('ServiceLogs').table('clustersServiceLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where resource_id has 'kusto-logs-7nt846zks6zp'\n        | where isnotempty(log.cid)\n        | distinct tostring(log.cid)\n        | limit 1\n    );\n    // Find all Maestro resource IDs for this cluster\n    let maestro_resource_ids = (\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n        | where isnotempty(resourceId)\n        | distinct resourceId\n    );\n    // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n    union\n        (\n            // Maestro logs: Resource bundle operations for this cluster and its associated resources\n            // Includes: resource creation, updates, deletions, status events, gRPC communications\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"maestro\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n            // Includes the CR reconciliation logs\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"hypershift\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        ),\n        (\n            // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n            // Includes: work agent operations, manifest application, status feedback\n            database('ServiceLogs').table('containerLogs')\n            | where timestamp between (_startTime .. _endTime)\n            | where namespace_name has \"open-cluster-management-agent\"\n            | extend logStr = tostring(log)\n            | where logStr has _cs_cluster_id\n            | project timestamp, container_name, msg = logStr, log\n        )\n    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T00:23:37.3079994Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\", ActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", SubActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\", ActivityStack:\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\"}, IsPermanent:false})\n    {\n        msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n        err: \u003c*errors.joinError | 0x11eea2518d38\u003e{\n            errs: [\n                \u003c*fmt.wrapError | 0x11eea31d1d20\u003e{\n                    msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                    err: \u003c*fmt.wrapError | 0x11eea31d1d00\u003e{\n                        msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                        err: \u003c*fmt.wrapError | 0x11eea2cd05c0\u003e{\n                            msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                            err: \u003c*fmt.wrapError | 0x11eea2cd0560\u003e{\n                                msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T00:23:30.3337685Z);\\nlet _endTime = datetime(2026-10-08T00:23:30.3337686Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-7nt846zks6zp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has _cs_cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has _cs_cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T00:23:37.3079994Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d\\\", ActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", SubActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"2dbee824-d49b-4a64-bb2b-a493f4fef7a1\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;b78e7740-aeae-43b4-b8e3-a48c2ee1f56d ARID=2dbee824-d49b-4a64-bb2b-a493f4fef7a1 \u003e GW.Http.CallContext/2dbee824-d49b-4a64-bb2b-a493f4fef7a1)\\\"}, IsPermanent:false})\",\n                                err: \u003c*v2.OneApiError | 0x11eea1622000\u003e{\n                                    ErrorMessage: {\n                                        Code: \"LimitsExceeded\",\n                                        Message: \"Request is invalid and cannot be executed.\",\n                                        Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n                                        Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n                                        Context: {\n                                            Timestamp: ...,\n                                            ServiceAlias: ...,\n                                            MachineName: ...,\n                                            ProcessName: ...,\n                                            ProcessId: ...,\n                                            ThreadId: ...,\n                                            ClientRequestId: ...,\n                                            ActivityId: ...,\n                                            SubActivityId: ...,\n                                            ActivityType: ...,\n                                            ParentActivityId: ...,\n                                            ActivityStack: ...,\n                                        },\n                                        IsPermanent: false,\n                                    },\n                                },\n                            },\n                        },\n                    },\n                },\n            ],\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107969278706716672","occurred_at":"2026-10-07T23:00:14Z","signature_id":"666d734da3ae47fa83d61d5609163ffa16d5c7b7a03ff4c22bce8466e9417d11","pr_number":6928,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"defe5d468c26c44c499810219b81a1dcd05b142f28a898794885f56e139df072","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-07T14:40:32.913Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-10-07T14:40:33.412Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-10-07T14:40:33.412Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1\ntime=2026-10-07T14:40:33.440Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 namespace=multicluster-engine\ntime=2026-10-07T14:40:33.491Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 dryRun=false\ntime=2026-10-07T14:40:33.491Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1\ntime=2026-10-07T14:40:33.534Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1\ntime=2026-10-07T14:51:01.640Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 err=\"failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-10-07T14:51:01.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107832305794945024","occurred_at":"2026-10-07T13:55:41Z","signature_id":"5313fb62a7a2ad329b1783c26de868383dd6a90d3fce29c37b41985c964d9b72","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.408450704225352,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-29T20:00:48Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","occurrences":1}],"full_error_samples":["time=2026-10-07T19:22:31.326Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-10-07T19:22:32.066Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values=\"map[enabled:true image:map[digest:sha256:7c0c84fdd1dde7d32754d602c60e7432108ecef850d5bcd2efc549110fbda458 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-10-07T19:22:32.066Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-07T19:22:32.374Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events namespace=monitoring\ntime=2026-10-07T19:22:32.684Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events dryRun=false\ntime=2026-10-07T19:22:32.684Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-07T19:22:32.822Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-10-07T19:27:38.910Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-07T19:27:38.911Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2107904302923124736","occurred_at":"2026-10-07T18:42:03Z","signature_id":"6b8d9984b55592851b0841c5e8c4a0f18f8f61520e7df4a51c7e6afc9a707725","pr_number":7361,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":4,"failed_runs":0,"raw_occurrences":0,"matched_occurrences":0,"runs_affected":0},"rows":null},{"environment":"prod","summary":{"total_runs":7,"failed_runs":7,"raw_occurrences":154,"matched_occurrences":153,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":51,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":51,"after_last_push_count":51,"trend_counts":[0,0,0,0,0,0,51],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z9xhpt9dxztr/cluster-candidate-4-22-8frl8d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc3c4040eec0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8frl8d: failed waiting for cluster=\"cluster-candidate-4-22-8frl8d\" in resourcegroup=\"rg-candidate-4-22-z9xhpt9dxztr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"name\": \"6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.12350471Z\",\n      \"endTime\": \"2026-10-07T11:13:04.942313802Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z9xhpt9dxztr/cluster-candidate-4-22-8frl8d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc3c4040eec0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8frl8d: failed waiting for cluster=\"cluster-candidate-4-22-8frl8d\" in resourcegroup=\"rg-candidate-4-22-z9xhpt9dxztr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"name\": \"6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.12350471Z\",\n      \"endTime\": \"2026-10-07T11:13:04.942313802Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2851e862a620\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ws854cfjbfc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"name\": \"5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:27.70826866Z\",\n      \"endTime\": \"2026-10-07T12:10:34.84968826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2851e862a620\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ws854cfjbfc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"name\": \"5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:27.70826866Z\",\n      \"endTime\": \"2026-10-07T12:10:34.84968826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231265410c40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6pl2gptcdfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"name\": \"3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:27.398564047Z\",\n      \"endTime\": \"2026-10-07T12:08:34.791319585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231265410c40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6pl2gptcdfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"name\": \"3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:27.398564047Z\",\n      \"endTime\": \"2026-10-07T12:08:34.791319585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"03da4f253bd1559e010815404447df7a21bfdb927bb20e898538689c3b1a7fc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"14135c0f42e6038fec848d3d13ac98a506c6bb5e44786249455cbf6831ee7ee5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"161f29459dd25be2b9948cb9b2928ee7923fcc1e8cff6ecba8619d7726e0f54d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"1882d241a3c9a7844d4783366ce6767db79330aaad9fc90ff0cbd2d6aa042b9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"222845611c5a37be3043cb1e73ff7a15f6703e7a0b75ca16f7f6d42711571715","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"22d4e8be87148da939695d13e1247fdbef5bf1728991abf73a2144aa28fb21d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"2a4544a43b4c4906031ddca838eac98d21615c334e36bf9f03833d3a9649ae82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"353b34662f60e8180bd22425317cae7ac8cb19040c048446d6da34eb0770224d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"38424e0b4e68892bec2eb668127066c6f2a26c76ecd81204afaf71f81b811e31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"3d2085352b10228c638ccaffe94249943e09d92267a10ea6e8a6f5acbc29dce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"3d70d948708128afed2253c50f9bf1c308631c17ba710f3522db7f18ef353d1c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"44595e30de386c5eb9f145b02f8c2bb5d743f010820664d1bfb5aebca2d405ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"481b8b75a1b96ca5473b5e3428bf844583807c6c42f822464b4a107e600e55af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"50ea19fbe8e7d37bc39003918df0f3db3330d2f50721b75d7305c5fee9e0c335","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"5401c8357fce4c3d57d0b54d9d0c1f712605e1965171f2bc7a7b99c403d39d17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"65aa41b9d85ed7b6471c3cae1678d3b40aedc84ad18e6c8bd418d159be210f5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"677239a911859f072d4ad2996b92ed87c2c9b33ef1b746a6f2c25221940cd869","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"6bb41cca9dbaab1d3f7795fa9b64ec70faea8e4b1d05cd73381f7f45ac480051","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"76f0209f03661a65125740c9733c3d4a11d4682178d1f51fbb322ae760b9115c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"81065ecc30a88dbe3690b63b71b1ab98da072a79a5e00eafdac69a5cf5e2e363","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"85089d90025f7e4b136612cbe3126cdeb157f81d89a6d1f85bf84e1242f59f52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"8b0eef9c1f0b24176611911df5aa0d480ab4a2120a5657f9316084eca9ea0bdf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"9612cc0cf025f89e9420879dadf6053baa6863d7bb5e5eaf07f761e2805945f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"96927fc5b20d35c1c9f5483135274e1e4c184f0687b9c7836cf8e11f8d5b485d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"977fcb1eb57ae1e979b4c1b950f7d9cc8abf8782c740f514024b50110304411e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"9f87a1d4862080e4d4b05c636d70abf9d6119ecf1dc420ad540a69449802aba5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a031589172d1628bbf1aaa7c92834693d2ad406fbe2d747a22903fa0d686e2c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a0a734eda8f093e83a4d94cde244e3c5aa94a9b640ad6a9042a19b8c1730494b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a1712b1999aa403be90c2c39032dbf3482175f4135dafb21660a6280946668f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"a2ce9a68f555401f462f07278b9c8957cd22840d85da390141086d71898ecd04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"aa67d6d32793cc52a0a7b7125b70972d054b6703e99dbe1985a2e305c66412a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ad1adaad6c931397a5cfb6188dbcbf7aec77f60cfab7d62edf2c9ea2f44522e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"b3c1cc9d18ffd5943aa3e4739d61d99e679706d5964b07a3b5374e971e51cc88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"b6c92859057095532f3f24d018d39c6d2961d7b695e023087e8f6547d3916117","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"b950484d8435e9ff010711daf067e8e3f2a694fad38348dda90d2c31aa435f5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"be2d920233867be6ec757af08451d598249bd0bcdda5bba21b83e5200e3fff3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"cec1415504f739ba8bb2e47520a81df319f676a287c871c00a9ca5c0bf800bf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"d514516b178028fea78395eeb242c18d910c33a636e520b12e6c30ec48120190","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"e6282f6819cbae90d2080bcc11689d8d54bca0907cf69387e75370c987b1244d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"e68e84c5ef68c6d5ed89d88a519e423d565bc4c0fe60d49c95bf93910fc3b5ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"e8b5cdeb0f741da0ec81fbee861c309ad3dc7a968fe5127637a89e53b5181949","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ea3cca5efa4a4d974d95cceb3e9779fac77cf63503170b6ca9e52bfc03e01dac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ecdadfc64076f4841dd66ae1fae57132eb211d25a58c181f0dbea68197273788","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ed9355468df05ca46faf44afb5f61337814d9c2ece35b6740bc25165e7398be6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f5875557a07249abfea079a118855841355a791d06199b1a53afdd74eb9ed76c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f62dfbfab386ba992afc971567b9d0d76b1b0e0237b5e0a68a1b3ec19e0c909f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f671365b45f055e398b3b3ea370acfc9dc7cbc2da9b1a10f010cc014e9b21b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"f952fed5c3234700f79a41bdd08cf3c1afc8a1f474d83944941014db01bd9d40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"fee1f7575ffa79a5f721c4a55f609c98395d74d67ae1e00d333ecc7ff0426314","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ff16efcd5d70df29c345cf8295a63cf7efb04beaae9b28016e8a9a1d93d36082","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"ff985ecaf6a3ec92db07f03052cef058d21f798d558e0c502a4afa2f5001396c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":43,"runs_affected":5,"failed_runs":5,"run_impact_percent":71.42857142857143,"anchor_occurrences":43,"after_last_push_count":90,"also_in":["DEV","STG"],"trend_counts":[6,5,1,12,1,6,43],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":35,"prior_last_seen_at":"2026-10-04T20:27:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ac110093d00\u003e: \n    failed to create NodePool np-gcl76m: failed waiting for nodepool=\"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\" in resourcegroup=\"rg-candidate-5-0-fvphxw8q5b7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"name\": \"891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:42:35.095507926Z\",\n      \"endTime\": \"2026-10-08T01:11:35.931061942Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ac110093d00\u003e: \n    failed to create NodePool np-gcl76m: failed waiting for nodepool=\"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\" in resourcegroup=\"rg-candidate-5-0-fvphxw8q5b7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"name\": \"891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:42:35.095507926Z\",\n      \"endTime\": \"2026-10-08T01:11:35.931061942Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3133a1380440\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dfm6bjrfc7jr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"name\": \"30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:37.937984296Z\",\n      \"endTime\": \"2026-10-08T00:36:41.067847774Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-nb26h-jsv7r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-nb26h-jsv7r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3133a1380440\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dfm6bjrfc7jr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"name\": \"30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:37.937984296Z\",\n      \"endTime\": \"2026-10-08T00:36:41.067847774Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-nb26h-jsv7r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-nb26h-jsv7r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36df90d3b540\u003e: \n    failed to create NodePool np-tlv9nh: failed waiting for nodepool=\"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\" in resourcegroup=\"rg-candidate-4-22-k8bbp9vgvcln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"name\": \"5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:39:43.973177646Z\",\n      \"endTime\": \"2026-10-08T01:08:45.917568185Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36df90d3b540\u003e: \n    failed to create NodePool np-tlv9nh: failed waiting for nodepool=\"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\" in resourcegroup=\"rg-candidate-4-22-k8bbp9vgvcln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"name\": \"5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:39:43.973177646Z\",\n      \"endTime\": \"2026-10-08T01:08:45.917568185Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"016fceb8857335fe4989864c4eb9b68117f9071b521a6c986d09b960f5ee4cc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"0df775ba92680196995c3176cc7ea2cf943ff930e220b5800c510ebe552cb1a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"2a467c8dce60d0e129d691c98b8512c8ec1ed634b0dd9e1fd39a756686e7121b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"4e1cdbd87ab7f2945023de5abed108f7e637f867d4aa7849637d211a4e777718","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"50f6bef9fb00580c65f782fcadba664ddc8335bf13557695ef58074a611962dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"89dcd307570448e824d023284861443a0dfbff7e5115cf1259f0fe9ffd4aa969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"c77f01f179024436a11b7e008e5544b959c8dce85fa7b4baaf4c0b19ae4e63ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"c8c0700c36a2dd608f6a26fbbe6a7b358648287bea29f72e02d30d5eb97939a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"d9a1da04dc652deaf64a62fa6192a041922c1cfe9c39fc4d3a8476773a01a71c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"1b24e8be23bf1c4a3a07ea52d5db1e9347b68ceaa63a98aa205db1be1d9e27db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"2f0ca01949f4ede477fc7bb9d9a950ce013da9ddd6123baf58a79a544d4837fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"5ad64e3bdfb8fb37ebf569989c2484a58f8d9a29ef2b587e78f90bbd9aca0db6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"b15f13ff28e10dd36674e727784b787d32b868277a3a5254568d42d9dddd74c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"cea78ce35fc90771f705f79a4927fdf38597e343e521b69fc7526b14eb7ab849","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"de05f0b33c084620030e8c696e91ec71531528875543bab2b0545455a55994aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"f7d1fcc30d377fd476b9f57741967c7af1149a2007cd215bbdac2b9536cded49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"04ad7125fe4905ae031c4007581e188577bf5010611a020b6a2efc132c5dc319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"753d143e88447561451de08944a9408e6f40d29e02738dd21a3764fd6cbe715c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"9785d8491224b970db4521504142e02625dbd60d73aaa543ecd4460be888e210","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"b80fe52aacb5620760c6bea3b3ed05432ded767100eb2b21c6c1e752e2a3acc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"bb8fa03a2e39c2209193639a0d46220b87e342b6202a3ad16c83d99f1ef37edf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"c7f5bdc4ebc923e20233f949e36f2ccb4fee458a453a1ee7a629440470bf56a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"cba8f12de6a543925775d61a4dc9325c74c4f0c58a30cea6a9fa01d63b96e361","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"dd2af2d4c434317800a4e02b03119effcc044182d265a3a22bcae48911b84b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"e3fb4a2b49a31e56c0902493d1ea2275a273a2c53bab17b6f360121094551afe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"522a7104067322270c1dd86403e8358453c5864b0e27aac4cc5b395bb3bb08b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"5cdf839fb7474b7c4643969bf85e742d6e7999ab72dceb75b236195841c8a7ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"92a56bbeee063c237c54c6a4c494c87e10a3cc5bf6ede5a63dcdfaf9e7e689fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"987d0ce54e6e4418fc185183edfbc093147759ab1307a6f8a59df037a4923be7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"9a80126b88b88f73aaed428e993ee3eb35ad2dc5ae4652821263df2cf667dc44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"d79148cbc396e3fb2417bb6a5c08546a4bc7626d2cb1d2b265e599d0466ee183","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"e8b902ffd47c497bcda3d3146c3d7891d7784d3563fc15636c6894b1f96e87aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"f60ae0ea29618f99864000653c981584b53aa5f616efd1c3c85d5021999710ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"fbd0c561558b517a4ab86f8ad9b18ffae8f602f9d4cdc462cbca34d38dfdb848","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"1ce8678d638300e56c4666b5590395913aef8694ae27b3df675cb7ad0f7b01b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"3f3489b7375ffb121a83750f4359bd32c3f7d88aea2c87381d44bdbba593595c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4116ee0556c26bd8833a3374c74c6a16c11f1c9d2170f9475869ca6784808061","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4af35e4cc7af67899aa17ed379f7e70c8cb4b98d18356ff405fe1baaa2a79cc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"6ce961a4bc8897fbd7b98df3cc5a58955ed5e747b18252fdfef55c7377a098c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"7f76fffd8b4b60d1d840bb53756f03a0b49f3555b816311695931c4a85b6abbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"ab88004ed09b9059f4902caf266cb767ff949a7b11999ba22a96ad8310ee0f0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"c29e28b214fa390074f60edb54d46b123550e1ba11cc9a9e14a2585e50f1d30d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e915fc8a853ff7f35cb4923c9b73fd715420c8f36391d017033ce5747e871487","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":17,"runs_affected":2,"failed_runs":2,"run_impact_percent":28.571428571428573,"anchor_occurrences":17,"after_last_push_count":89,"trend_counts":[0,6,15,16,17,18,17],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-04T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p2kwgg\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd94e927e240\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-xqwwvdsh9nb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p2kwgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p2kwgg\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd94e927e240\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-xqwwvdsh9nb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p2kwgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-dl424r\" to 4.23.0-0.nightly-multi-2026-10-06-030402\nUnexpected error:\n    \u003c*exported.ResponseError | 0x23e2acc0cbd0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-kjs4dvf659k9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-dl424r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-dl424r\" to 4.23.0-0.nightly-multi-2026-10-06-030402\nUnexpected error:\n    \u003c*exported.ResponseError | 0x23e2acc0cbd0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-kjs4dvf659k9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-dl424r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2661dd4d00c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-z8pk7tmfp9jj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2661dd4d00c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-z8pk7tmfp9jj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0f0a376071a48cbdd5611a1be04be8ee77bb833d681538cdad43fbb56b08abe9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"1ebec80e3d431198968e72632366b202903ff8ad592e0ab7fed494102db1b4fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"4894e3e856cbc512e113a81c6782b48eb1a23170d9613bd7146b82b50a5ba5be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"4fbf30b74782eb5ddefc289814bdb46d277c794e317bb41e0d934ac244ef4a6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6a259fc11a58d55268cdd102b80e33c281baa34d38f39e72cecc21167dfcf5fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"764d2f0ddbacda8f6566b981d59dc30f4bcdb0c97dd732f715407bf107cec2b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"8048cfa91100e6e8b8333a3eeebf145761ee4c1f1cf7dbde205df021632e34c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"f9758d5d36fc27455b6a21116c9b88ebed4012752a67056213ed40b7fb482538","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"1cbf26ee1e5ab30922ecc4509d85eefbb36e982c6ef1ac8f2183590c4796adf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"3b196b8c142b7de81731bf3edd13cc8231edfe7f5ca4159cff7cc65b94f4ec03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4cb77a8e2eb04e51d1e31b43fc33204e1adae88dabf9e8306d0b10cbba5bc76e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"5d7b068f725ac38a9e9614dfdb837b0fb5f432769ef58cb2757748484e960a34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"7011579edb7da9f880b9ea345fd4c99a061064cbde4affb3e7853dd76feb67c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9338ca7a8b3338ded2c268d7f0328d1f40dc0524254e97f8e79312c850e87765","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"cf3da2b8d691239f5c5c38c7c621ba3e4f4593ae21d6761ff8aa4453720a3201","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"f3a567f87a32f64da957df0785589b6bd3c47799e90deadbfe5bc1b927ed369a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"f88443bde247b05fc776f561d632239634a07c67d0f9534f9fadfe4591ff0ae7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":12,"runs_affected":2,"failed_runs":2,"run_impact_percent":28.571428571428573,"anchor_occurrences":12,"after_last_push_count":60,"trend_counts":[0,2,11,11,12,12,12],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-04T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fa024921260\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-8hhqrj\" resourcegroup=\"rg-np-dg-minor-8hhqrj-7f8bnzmkm895\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-8hhqrj-7f8bnzmkm895/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-8hhqrj/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fa024921260\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-8hhqrj\" resourcegroup=\"rg-np-dg-minor-8hhqrj-7f8bnzmkm895\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-8hhqrj-7f8bnzmkm895/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-8hhqrj/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e46f700980\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tdmf8r24g52m\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-tdmf8r24g52m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e46f700980\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tdmf8r24g52m\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-tdmf8r24g52m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35088dfe27a0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-dwmb74mwp62t\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-dwmb74mwp62t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35088dfe27a0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-dwmb74mwp62t\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-dwmb74mwp62t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"140752297cfc5196913f69b2f55f75602e169015ac1cf424a20aa497281b774b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"43468b889804001bbbe14bf435f20f53ae868359b8d38f9fb0a8ca86f38b3a14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"5f39b424b8f2fd154227943af3c6cc49910c69a6a961a7b0c48ca3a834eee8a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"0058e792d032ae53f5bca2e179ae1c409cdd5b3c2bb64768e1eb44b0951a5c10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4cec21a3bb1485fbbf2fb4c6307e0674539ae34b252d1fd61badf2ee1e7ee5b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"5be1160c2fa84fb7c2ea3eb2b520a681bba0ca7197c4252afbb61a475fb1097d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"7577f160ea754982b08b2f90867551e7ee2355fdac02ed66e99f5880127d9f79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"8696e2e81d9653753fa26ee12fc06824880c95acb852d196ef0af32ebd7fd0a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"8cf036c6672baf731fc796bc4ac50089b693a6a516c1216bf1d9e70f531eaf90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e68cdd216d0150edd847f603ef682998ad7b0271775710e0858f3f7f5b37530f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"f8f6ef862ba076aa00ce8d20e4b36cddbf5813529ac198d1dd60133ef734dda8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"fac4ce6bbb505620cdc94744fa17f94c3c1fa837421bd30a8671b97b8cca987e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":7,"after_last_push_count":28,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-24T02:00:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186c1f59fe00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-xkrgtz49hrh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xkrgtz49hrh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186c1f59fe00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-xkrgtz49hrh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xkrgtz49hrh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e92b4ffb60\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jsrm64gc6pjs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jsrm64gc6pjs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e92b4ffb60\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jsrm64gc6pjs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jsrm64gc6pjs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1d4613307440\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-xccktj7qbpzs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1d4613307440\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-xccktj7qbpzs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"2a9264902cac7135ae10487055f2ec90d442d6082711c47d54f8a83eaf00eefd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"4285d36e41c52c52f14f33e255d32a9fbefe892fb4555c01f3990db62c8bb7f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"554cf56528558edb3b288f90dec934fc7fd2db28015f4ef2d61362bc08676deb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"67e3544022c911074f09ce65702282c6041cff3d54247562a71ba2e779489a2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"aa6f57794e5ffec707db8f8f3922b0d34c04ee60936df21bade7d3f757093a83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"d049db17883d8e9b0b2cefc43813652d974c416c9ec136721194aeabfe76194d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"dd8473cf389dd6f494be575174dc277ffa211b0aa8afa70458bd3be0e7b014db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":71.42857142857143,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":27,"prior_last_seen_at":"2026-10-04T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32955c70b520\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f92csqngd5f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"name\": \"c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:32:50.695442989Z\",\n      \"endTime\": \"2026-10-08T02:01:50.773194309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32955c70b520\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f92csqngd5f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"name\": \"c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:32:50.695442989Z\",\n      \"endTime\": \"2026-10-08T02:01:50.773194309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1c11c2b2b440\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-npbf7r72xkcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"name\": \"26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:01:08.520978007Z\",\n      \"endTime\": \"2026-10-08T00:30:17.207038187Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1c11c2b2b440\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-npbf7r72xkcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"name\": \"26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:01:08.520978007Z\",\n      \"endTime\": \"2026-10-08T00:30:17.207038187Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d0dde8c3780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cdl8czz92fjt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"name\": \"1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T21:30:13.060183446Z\",\n      \"endTime\": \"2026-10-07T21:59:13.201730463Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d0dde8c3780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cdl8czz92fjt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"name\": \"1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T21:30:13.060183446Z\",\n      \"endTime\": \"2026-10-07T21:59:13.201730463Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"ad8aa7d0e446ca97b82f47c75fb4959ef7171a8fc84389bd39c3be3472d4a550","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"f06515c3d076448409dc5ad1d9d943a2b21906ced720a3fc9685ab778dec06cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"19f8dd197db68f57fa6b4eeb4e85cec78926fb3e66586ba3c7b4f6152ea72984","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"29adca326faa1bc8f35a40a7651e6edc98aa900d6b8aa8c789d1b2db2ab468ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"dcfadc4ebeef20592def4e3d8be48fe15094c12bcb500f3186b13d44a79114cc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":57.142857142857146,"anchor_occurrences":5,"after_last_push_count":7,"also_in":["DEV","STG"],"trend_counts":[0,1,0,0,0,1,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-02T23:28:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30a375627b20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30a375627ae0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x30a375627a80\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30a375627b20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30a375627ae0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x30a375627a80\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2c512700e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bsqjlrm7bfgr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"name\": \"597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:05.991580079Z\",\n      \"endTime\": \"2026-10-08T00:36:11.101154752Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2c512700e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bsqjlrm7bfgr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"name\": \"597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:05.991580079Z\",\n      \"endTime\": \"2026-10-08T00:36:11.101154752Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x42d11f950c8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2vrv7v7vlpcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"name\": \"9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:08:33.943284812Z\",\n      \"endTime\": \"2026-10-07T20:37:40.405868199Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x42d11f950c8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2vrv7v7vlpcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"name\": \"9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:08:33.943284812Z\",\n      \"endTime\": \"2026-10-07T20:37:40.405868199Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"aaecb179fa0c8619e61820f2147630fb08f8bc80cc1c2f7237571a27bd0b49f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"d5adfea4d2420279717d9c7a8651e85415083eda53a1fc1cec731d467be31793","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","signature_id":"42cc3045165e80f27d068734529151281e6d04adb423b2dda7c90a129047ebf7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","signature_id":"02820abf0337fcd30dd807581ed2e7cced4ca7f03dbbf80f30a8a902b7db9b1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"45693c845a913fd4511bed4e661c2d3c6bfa8a4cc05cb75085a24072104e134b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":4,"after_last_push_count":19,"trend_counts":[0,5,3,2,3,2,4],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-04T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-gcjnnr\" on minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2862c3242150\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-97btfzzdk5qt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-gcjnnr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-gcjnnr\" on minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2862c3242150\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-97btfzzdk5qt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-gcjnnr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-wfqdx6\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x35670f110a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-m89bbfgxxvxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-wfqdx6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-wfqdx6\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x35670f110a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-m89bbfgxxvxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-wfqdx6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-p4lgwf\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1b0947b50a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-2gpxkvcjbvk8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-p4lgwf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-p4lgwf\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1b0947b50a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-2gpxkvcjbvk8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-p4lgwf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4aadc6cfa9574f8e73cbcf281be4f06712a54bf21ebdbda8d42084548d72bf1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"b4c938ffd1e07317c7f1a8b66bdd6168fa2ea345151308900639a09f23bc84fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"bbea7e2da871e62baa803b44294e1265be218d3de0dc6cf6f671b9fcb2dfd871","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","signature_id":"eca13765fa9405f95252cf15d7ab0f204511f501fe8006308939ad15b76113e0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":3,"after_last_push_count":71,"trend_counts":[2,2,2,2,3,3,3],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":21,"prior_last_seen_at":"2026-10-04T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x312a48ea2980\u003e: \n    failed to create NodePool np-zgtwtf: failed starting nodepool creation \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\" in resourcegroup=\"rg-nightly-4-20-j42q4dvn7t74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-j42q4dvn7t74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zgtwtf/nodePools/np-zgtwtf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x312a48ea2980\u003e: \n    failed to create NodePool np-zgtwtf: failed starting nodepool creation \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\" in resourcegroup=\"rg-nightly-4-20-j42q4dvn7t74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-j42q4dvn7t74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zgtwtf/nodePools/np-zgtwtf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32c6fec798e0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-vbjww8\" in resourcegroup=\"rg-np-skip-minor-vbjww8-9dlhkbhpd9hb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vbjww8-9dlhkbhpd9hb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vbjww8/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32c6fec798e0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-vbjww8\" in resourcegroup=\"rg-np-skip-minor-vbjww8-9dlhkbhpd9hb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vbjww8-9dlhkbhpd9hb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vbjww8/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1829881ca3e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7pmxvz\" in resourcegroup=\"rg-np-version-upgrade-7pmxvz-jjjscx2plvvj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pmxvz-jjjscx2plvvj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pmxvz/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1829881ca3e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7pmxvz\" in resourcegroup=\"rg-np-version-upgrade-7pmxvz-jjjscx2plvvj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pmxvz-jjjscx2plvvj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pmxvz/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6037211c53fad4b43e675ddfe0a29c9bf8bd819adc8e5ef742dd4a3d5eac5fbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"84b2c7e168966ee41c9321960df1699f3c7e6ff0bbf13ad9815db6fc66c666f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"95caab3bc51d32ccb091310bdf33b3e84812faa9fc5c890de7a18f95ac7e465e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":2,"after_last_push_count":80,"also_in":["STG"],"trend_counts":[5,2,2,2,2,2,2],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":20,"prior_last_seen_at":"2026-10-04T23:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x26d333bbc300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-ckbnsnb4ttgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x26d333bbc300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-ckbnsnb4ttgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x6c1c704d590\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6dmndbpnqjgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x6c1c704d590\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6dmndbpnqjgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"68d4ce843ab001efd587a8e4a37b0a8261c4ad82e127c6e95cfebd9d1f8cff44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"b614f8950b85c4c4d2722a409ef292ea389bae31115c3e2b81722d59a537583b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,1,1,0,1,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-03T23:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x2e5cf51ae018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2e5cf5120ca0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf5120c80\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4986960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2e5cf52a2040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf52a2020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4b086e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x2e5cf51ae018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2e5cf5120ca0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf5120c80\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4986960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2e5cf52a2040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf52a2020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4b086e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","signature_id":"d4720f51184f2ea80269323a34de851f7131f4cdc47aa604ea02b7639bfb5d5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6414ac0ffcaccf7efef39a38c0b45dd9d34701a344992b34e8e112eb6e3a27b7","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x99270993880\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x99270993880\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","signature_id":"1b1d10ece3b90ada6d195629824ea959318da72c37d7910b78e8868c3143e501","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-18T17:02:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2fbb237581e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kc94rh5nhxb8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2fbb237ab310\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2fbb237581e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kc94rh5nhxb8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2fbb237ab310\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","signature_id":"6a9c2523f9881cd28e189784f3244e2b462a82a81d6d904e71798161eadd2b44","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":61,"matched_occurrences":60,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":40,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":40,"after_last_push_count":104,"also_in":["DEV","PROD"],"trend_counts":[6,5,2,1,1,18,40],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":32,"prior_last_seen_at":"2026-10-04T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e843b97f120\u003e: \n    failed to create NodePool np-xkh7k5: failed waiting for nodepool=\"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\" in resourcegroup=\"rg-nightly-4-22-lnvbhdz2bflf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"name\": \"dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:40:08.180826205Z\",\n      \"endTime\": \"2026-10-08T00:09:12.758937229Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e843b97f120\u003e: \n    failed to create NodePool np-xkh7k5: failed waiting for nodepool=\"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\" in resourcegroup=\"rg-nightly-4-22-lnvbhdz2bflf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"name\": \"dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:40:08.180826205Z\",\n      \"endTime\": \"2026-10-08T00:09:12.758937229Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a93c5372000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v2qsz6wkc724\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"name\": \"93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:29.82535222Z\",\n      \"endTime\": \"2026-10-07T23:46:38.16324507Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a93c5372000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v2qsz6wkc724\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"name\": \"93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:29.82535222Z\",\n      \"endTime\": \"2026-10-07T23:46:38.16324507Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bb2d5ca8760\u003e: \n    failed to create NodePool np-dhdr4d: failed waiting for nodepool=\"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\" in resourcegroup=\"rg-nightly-5-0-w2nf95ndpzrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"name\": \"5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:07.401401662Z\",\n      \"endTime\": \"2026-10-08T00:12:12.648695446Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bb2d5ca8760\u003e: \n    failed to create NodePool np-dhdr4d: failed waiting for nodepool=\"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\" in resourcegroup=\"rg-nightly-5-0-w2nf95ndpzrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"name\": \"5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:07.401401662Z\",\n      \"endTime\": \"2026-10-08T00:12:12.648695446Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0d3977ea4a90af04720c42d0bb759a3fb96691ce43398b60e6eef15c0a367212","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"35c1bfad26ec391bd6694018ed5a8d59c0813fc87ac12982198a869b9604acb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"68cb7350eeb61710d23cb67024cbbf9383969e0663d83204c8bdbdff0d33b967","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"698107089cf97b784786a6213e09a0a68dc2d0bd01e69c8e0e0c0fdb8f948aab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"8ee2deee6d590c8c6f98a003b663ac08a67a9a1ecc4cc6560bd842c5b1c9a52d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"a0dba5d1188bd276353fe4d2637a368162095ec26a8c24aee58b4ef68e6b2355","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"d3f83bccde47a3e6aa4acd1828dd9099b38db2477a89ab6bee0285d81eb16b5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"01163b61c17b562a500c04958cb33fe485d2c7ca115bc405d91028b41fee31fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"30c2dae7bbb702188aef5ab4608ef49e36dd251396ec6996c25febee6fad61be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"62b9a0dbb94dec0c48d4230d8c8526f7eb93e53eaeef32b48228dd8d42af77f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"6e47728030dcf1f6d034b909c5fda78d2f12b066fb41e2f513f61773969a2a38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"70ad7e4081cf1f599d09c2f83be92ab81460d5e3baf40247b0aaf860738f9066","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"bd4230f06a85eb9e1292a6291f9776d671fb62ec21c099d1d8dbdbd83275c152","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"cf1becd17e665f40e312fb2fb9948d50c9c5fa62d4421b6733acabe9bde7f248","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"ede7e9813082bc8cecccb538add5f1a3c53fe9bae98639f390f3a1e3e9e834e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"0295e7035dfce1af760e5a9f96d8886860036c81cb7a349f83dfe84f97f0ab25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"2efd6126d606d5f6f80d0316433b168030b43d3ea235668b13059c9b830d3aa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"660518fdfbb9f0800a819d55d8bc3171022d1eadfaf9a16981cf1f090bdaf04c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"844a548f21a121190f4b76ba3514e970ff80d7128375752094c4e416f2fd6f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"88363a2b12377ced6544609973aca118fa1d1df45bf3c3b0605e39c27653faec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"8a41ca55dc9a29f3bb19bad80e28580eee3b1db49cbcab92f2d225a074105caf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"a131321bd10141e15cfba687cd99d26f8cc540813c1b560df836c2ddf028dca9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"c1fc956072a43b79a620c5e7511a401ac3079dfd2ffe4780e4315cf518f89d75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"ffa11b5d4792edceb448fa6645e938d4beba1a8400ac52ce970668a6ebf44a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"42d95229988ca74bd7cb337361282d8c16d787466292deaf2232070c3cef245e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"636c0e8426e9e41af610392dfe2585499dc4ca5ac63cc2c2ad03af827766f74a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"6afa453b179354544c5915baf6f73ad97a3103ed41f793200617265657041048","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"ace0a1418b7953740f38672a49af405998a9e3e24c4a33b95e0d999dd68228b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"b2b0c546047cd7f8dfbc1e8fd084cbf9b0c93bccae10e310d13b26a94be9f023","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"d41cf6f362daae9f0f271e170de364045dff8bca3bf030691011bad678666f7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"def278b028e2cfc71409628aa852a649d7df162da0e1ca879d80fffe9d9e1dd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"f6435cbef4dc6ebe4912bc5681a94ef53c08760658835c2e2be55f09f269435c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"0af96c9de788a43bfc36cc28d4ef2f7ed16e382ed3dce340c647a636af5ecb10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"2d1db9807c28b66174164739cafde112b3198801994efbfaf7eb91ee2bc663c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"461d3cd85376afaa0deb74d80a0b8c0bf44e6e13b152912c7fa7055c0a01ba52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4acdd568139e3c3c134edd0d6f256da22e4a7a8368cc84137523b794cc84f751","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"87cadb376d6cd5fe3148fe99a8a0a312ae518ce1cd8d34338748d862f979c65e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"90e197c419ca4c3d130ecefd2a74215194ffd5df0642d2dc2d4c34686fc824f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9264fae13d8a0e126024fe87016ca0eb955eeacc351b1d83a0e62b7b2d4f8888","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e3312dc69a51a671f116346996b6b0e62dfdfde1bc0d102af6a4d41fef2b5576","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":8,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":8,"after_last_push_count":9,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,1,0,8],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26a27898a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9xrgvjx8lwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"name\": \"e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:33.092346932Z\",\n      \"endTime\": \"2026-10-07T23:46:38.337633525Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26a27898a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9xrgvjx8lwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"name\": \"e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:33.092346932Z\",\n      \"endTime\": \"2026-10-07T23:46:38.337633525Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x390344d20c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4wvdmxwppl4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\",\n      \"name\": \"02042050-d676-4d90-bee1-de9c86994c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:29.815037524Z\",\n      \"endTime\": \"2026-10-07T14:59:36.125232922Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x390344d20c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4wvdmxwppl4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\",\n      \"name\": \"02042050-d676-4d90-bee1-de9c86994c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:29.815037524Z\",\n      \"endTime\": \"2026-10-07T14:59:36.125232922Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb98d7c3a108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6z9fvf852z99\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"name\": \"93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:19.676644212Z\",\n      \"endTime\": \"2026-10-07T14:59:26.138957659Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb98d7c3a108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6z9fvf852z99\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"name\": \"93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:19.676644212Z\",\n      \"endTime\": \"2026-10-07T14:59:26.138957659Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6ba651d962b63ca9737c83935c51fabd9671a4349db5bc958c0be99ef420497d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"38e91e14bbfa3b78d5e999c68fea443396976fc0d1daad98d4cf67b9eaa12e1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"f9b070f99215534333fb93adb09f749de178f54c1544c512656282297ceb7a23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"f9af646ef53adca9c38d8c7a82da5f67f58665aa87755f378eaeda7b1930de7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"255e1dca1572a70b8f0d0f5f6aa345989e66a6b4f6f9e32911b805b3bfa7a379","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"c77d4d78f4573df1f5f58548e67fdb600dd43ca5bb56e96677224027c975bc44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"06edfc93dd116cf1f2412df4933dab83c6d60f345bd6545be6187369b744fbb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9174db2c962c5b14b002b9d9fc7ad502cc024cdfc29401bc9447ace098a0ae20","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bxmr827d8pqp\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":24,"prior_last_seen_at":"2026-10-04T22:01:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x115d2dac76c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-knjdfr9bnmx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"name\": \"3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:45:51.426276166Z\",\n      \"endTime\": \"2026-10-08T00:14:52.690397112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x115d2dac76c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-knjdfr9bnmx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"name\": \"3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:45:51.426276166Z\",\n      \"endTime\": \"2026-10-08T00:14:52.690397112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x25e8dd52bee0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tkm2qrpq9mx2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"name\": \"b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T15:25:44.42195423Z\",\n      \"endTime\": \"2026-10-07T15:54:45.869722515Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x25e8dd52bee0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tkm2qrpq9mx2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"name\": \"b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T15:25:44.42195423Z\",\n      \"endTime\": \"2026-10-07T15:54:45.869722515Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x28b27f4fd8c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-flmg4dn4tndz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"name\": \"8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:07:13.045361342Z\",\n      \"endTime\": \"2026-10-07T11:36:15.865514346Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x28b27f4fd8c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-flmg4dn4tndz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"name\": \"8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:07:13.045361342Z\",\n      \"endTime\": \"2026-10-07T11:36:15.865514346Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"41911d09cb3faeb31e1614a3ce95b71553ee40f58fbdbb9b2b3538c30fc780cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"e1898cdee251c0de6f399c876a928fe1ac01341d45cd38e4d6b00e9b3fb43608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"fbe36cbbceb1d0e1025eb3b54fd10fe074a1d442f305fcf1dd1cb97792c28108","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"c701b9c1b5bb023dff9bd4abb1de8fbc2c5ce5288ba7fb66eaf94d47e43c2e87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"6cb55ad6e469d73beffb431fa7679fc0430ca42fd9f6b8f1a796395bc2d38f01","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":52,"also_in":["PROD"],"trend_counts":[0,2,5,0,5,5,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-03T23:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x27f249e4c7e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-w7l7gcf22jkn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x27f249e4c7e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-w7l7gcf22jkn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1512977bb1a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qrbwmhc6tvhs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1512977bb1a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qrbwmhc6tvhs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3dbe3c0729c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-g2tcvfj4mtzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3dbe3c0729c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-g2tcvfj4mtzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0660a8b07b67f6039cafbd2db5de6076302394cf9033181aa61d0e1bed619a61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"24834a656a673fd3cfb5ae1fccf25d73d8b576895e93c65a3c6a94a528913c38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"748728f92e96c01a73c8dc01210faf4d617f86fb186b83b70b3031c704b1a940","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"79bf72a228ccd77d5a3ed3ea923d269a222d78760e0d3456d70db4a2a4ed54ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"abb5a95e8ea83117472e6c379d204e8f2c25f4bdb19bf5b0daa764dda033af3b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5d4aaaaae5c748327f53adcfa681b6d36f3f8da08e1b106abec14dbcf1f183c1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down cluster resources, managed cluster, hosted cluster, namespaces; [descendantResources] remaining resources: applydesires, externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27901568a690\u003e: \n    resource group \"external-auth-cluster-ch86qj77dnn4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ch86qj77dnn4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"name\": \"0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T06:22:44.810261014Z\",\n      \"endTime\": \"2026-10-07T06:46:52.177879454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x279015820520\u003e{\n                msg: \"resource group \\\"external-auth-cluster-ch86qj77dnn4\\\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0x279015820460\u003e{\n                    msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*fmt.wrapError | 0x279015820440\u003e{\n                        msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                        err: \u003c*fmt.wrapError | 0x279015820420\u003e{\n                            msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            err: \u003c*exported.ResponseError | 0x27901606a960\u003e{\n                                ErrorCode: \"InternalServerError\",\n                                StatusCode: 200,\n                                RawResponse: {\n                                    Status: \"200 OK\",\n                                    StatusCode: 200,\n                                    Proto: \"HTTP/2.0\",\n                                    ProtoMajor: 2,\n                                    ProtoMinor: 0,\n                                    Header: {\n                                        \"Cache-Control\": [...],\n                                        \"X-Envoy-Upstream-Service-Time\": [...],\n                                        \"X-Ms-Operation-Identifier\": [...],\n                                        \"X-Ms-Correlation-Request-Id\": [...],\n                                        \"X-Msedge-Ref\": [...],\n                                        \"Pragma\": [...],\n                                        \"Content-Type\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [...],\n                                        \"Strict-Transport-Security\": [...],\n                                        \"X-Content-Type-Options\": [...],\n                                        \"Date\": [...],\n                                        \"Content-Length\": [...],\n                                        \"X-Ms-Request-Id\": [...],\n                                        \"X-Ms-Routing-Request-Id\": [...],\n                                        \"X-Cache\": [...],\n                                        \"Expires\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [...],\n                                    },\n                                    Body: \u003c*exported.nopClosingBytesReader | 0x279015820220\u003e{\n                                        s: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        i: 0,\n                                    },\n                                    ContentLength: 1273,\n                                    TransferEncoding: nil,\n                                    Close: false,\n                                    Uncompressed: false,\n                                    Trailer: nil,\n                                    Request: {\n                                        Method: \"GET\",\n                                        URL: {Scheme: ..., Opaque: ..., User: ..., Host: ..., Path: ..., Fragment: ..., RawQuery: ..., RawPath: ..., RawFragment: ..., ForceQuery: ..., OmitHost: ...},\n                                        Proto: \"HTTP/1.1\",\n                                        ProtoMajor: 1,\n                                        ProtoMinor: 1,\n                                        Header: {...: ..., ...: ...},\n                                        Body: nil,\n                                        GetBody: nil,\n                                        ContentLength: 0,\n                                        TransferEncoding: nil,\n                                        Close: false,\n                                        Host: \"management.azure.com\",\n                                        Form: nil,\n                                        PostForm: nil,\n                                        MultipartForm: nil,\n                                        Trailer: nil,\n                                        RemoteAddr: \"\",\n                                        RequestURI: \"\",\n                                        TLS: nil,\n                                        Cancel: nil,\n                                        Response: nil,\n                                        Pattern: \"\",\n                                        ctx: \u003csuppressed context\u003e,\n                                        pat: nil,\n                                        matches: nil,\n                                        otherValues: nil,\n                                    },\n                                    TLS: {\n                                        Version: 772,\n                                        HandshakeComplete: true,\n                                        DidResume: false,\n                                        CipherSuite: 4866,\n                                        CurveID: 25,\n                                        NegotiatedProtocol: \"h2\",\n                                        NegotiatedProtocolIsMutual: true,\n                                        ServerName: \"management.azure.com\",\n                                        PeerCertificates: [..., ..., ...],\n                                        VerifiedChains: [...],\n                                        SignedCertificateTimestamps: nil,\n                                        OCSPResponse: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        TLSUnique: nil,\n                                        ECHAccepted: false,\n                                        HelloRetryRequest: true,\n                                        ekm: 0x585e900,\n                                        testingOnlyPeerSignatureAlgorithm: 2052,\n                                    },\n                                },\n                                errMsg: \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27901568a690\u003e: \n    resource group \"external-auth-cluster-ch86qj77dnn4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ch86qj77dnn4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"name\": \"0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T06:22:44.810261014Z\",\n      \"endTime\": \"2026-10-07T06:46:52.177879454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x279015820520\u003e{\n                msg: \"resource group \\\"external-auth-cluster-ch86qj77dnn4\\\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0x279015820460\u003e{\n                    msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*fmt.wrapError | 0x279015820440\u003e{\n                        msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                        err: \u003c*fmt.wrapError | 0x279015820420\u003e{\n                            msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            err: \u003c*exported.ResponseError | 0x27901606a960\u003e{\n                                ErrorCode: \"InternalServerError\",\n                                StatusCode: 200,\n                                RawResponse: {\n                                    Status: \"200 OK\",\n                                    StatusCode: 200,\n                                    Proto: \"HTTP/2.0\",\n                                    ProtoMajor: 2,\n                                    ProtoMinor: 0,\n                                    Header: {\n                                        \"Cache-Control\": [...],\n                                        \"X-Envoy-Upstream-Service-Time\": [...],\n                                        \"X-Ms-Operation-Identifier\": [...],\n                                        \"X-Ms-Correlation-Request-Id\": [...],\n                                        \"X-Msedge-Ref\": [...],\n                                        \"Pragma\": [...],\n                                        \"Content-Type\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [...],\n                                        \"Strict-Transport-Security\": [...],\n                                        \"X-Content-Type-Options\": [...],\n                                        \"Date\": [...],\n                                        \"Content-Length\": [...],\n                                        \"X-Ms-Request-Id\": [...],\n                                        \"X-Ms-Routing-Request-Id\": [...],\n                                        \"X-Cache\": [...],\n                                        \"Expires\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [...],\n                                    },\n                                    Body: \u003c*exported.nopClosingBytesReader | 0x279015820220\u003e{\n                                        s: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        i: 0,\n                                    },\n                                    ContentLength: 1273,\n                                    TransferEncoding: nil,\n                                    Close: false,\n                                    Uncompressed: false,\n                                    Trailer: nil,\n                                    Request: {\n                                        Method: \"GET\",\n                                        URL: {Scheme: ..., Opaque: ..., User: ..., Host: ..., Path: ..., Fragment: ..., RawQuery: ..., RawPath: ..., RawFragment: ..., ForceQuery: ..., OmitHost: ...},\n                                        Proto: \"HTTP/1.1\",\n                                        ProtoMajor: 1,\n                                        ProtoMinor: 1,\n                                        Header: {...: ..., ...: ...},\n                                        Body: nil,\n                                        GetBody: nil,\n                                        ContentLength: 0,\n                                        TransferEncoding: nil,\n                                        Close: false,\n                                        Host: \"management.azure.com\",\n                                        Form: nil,\n                                        PostForm: nil,\n                                        MultipartForm: nil,\n                                        Trailer: nil,\n                                        RemoteAddr: \"\",\n                                        RequestURI: \"\",\n                                        TLS: nil,\n                                        Cancel: nil,\n                                        Response: nil,\n                                        Pattern: \"\",\n                                        ctx: \u003csuppressed context\u003e,\n                                        pat: nil,\n                                        matches: nil,\n                                        otherValues: nil,\n                                    },\n                                    TLS: {\n                                        Version: 772,\n                                        HandshakeComplete: true,\n                                        DidResume: false,\n                                        CipherSuite: 4866,\n                                        CurveID: 25,\n                                        NegotiatedProtocol: \"h2\",\n                                        NegotiatedProtocolIsMutual: true,\n                                        ServerName: \"management.azure.com\",\n                                        PeerCertificates: [..., ..., ...],\n                                        VerifiedChains: [...],\n                                        SignedCertificateTimestamps: nil,\n                                        OCSPResponse: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        TLSUnique: nil,\n                                        ECHAccepted: false,\n                                        HelloRetryRequest: true,\n                                        ekm: 0x585e900,\n                                        testingOnlyPeerSignatureAlgorithm: 2052,\n                                    },\n                                },\n                                errMsg: \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"bbd40a695906a4219c65ee7a3a59b31da7d9339ef823acf986d08e0c6c81eb03","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"88cee2a035a0f46609d44b8db5480aa9ed0efd1c5fec7797c0cbce1364777b87","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2655580ad6c0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2655580ad6c0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"28108545b283b909283d0a8f87e72879ae1a1db48c35cd0650ff7b4851a09bcb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}